cryptic-io
/
cryptic-net
3
0
Fork 0
Code Issues 1 Pull Requests Projects Releases 3 Activity

Use a real private key for garage instances

Browse Source
main
Brian Picciano 7 months ago
parent b26f4bdd6a
commit 711d568036
9 changed files with 164 additions and 215 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 12
      entrypoint/src/bootstrap/garage.go
  2. 5
      entrypoint/src/bootstrap/hosts.go
  3. 8
      entrypoint/src/cmd/entrypoint/daemon_util.go
  4. 66
      entrypoint/src/cmd/entrypoint/garage_util.go
  5. 87
      entrypoint/src/garage/garage.go
  6. 41
      entrypoint/src/garage/infinite_reader.go
  7. 101
      entrypoint/src/garage/infinite_reader_test.go
  8. 55
      entrypoint/src/garage/peer.go
  9. 4
      entrypoint/src/garage/tpl.go

12
entrypoint/src/bootstrap/garage.go
Unescape View File

@ -12,9 +12,9 @@ const (
)
// GaragePeers returns a Peer for each known garage instance in the network.
func (b Bootstrap) GaragePeers() []garage.Peer {
func (b Bootstrap) GaragePeers() []garage.RemotePeer {
var peers []garage.Peer
var peers []garage.RemotePeer
for _, host := range b.Hosts {
@ -24,7 +24,8 @@ func (b Bootstrap) GaragePeers() []garage.Peer {
for _, instance := range host.Garage.Instances {
peer := garage.Peer{
peer := garage.RemotePeer{
ID: instance.ID,
IP: host.Nebula.IP,
RPCPort: instance.RPCPort,
S3APIPort: instance.S3APIPort,
@ -50,13 +51,14 @@ func (b Bootstrap) GarageRPCPeerAddrs() []string {
// ChooseGaragePeer returns a Peer for a garage instance from the network. It
// will prefer a garage instance on this particular host, if there is one, but
// will otherwise return a random endpoint.
func (b Bootstrap) ChooseGaragePeer() garage.Peer {
func (b Bootstrap) ChooseGaragePeer() garage.RemotePeer {
thisHost := b.ThisHost()
if thisHost.Garage != nil && len(thisHost.Garage.Instances) > 0 {
inst := thisHost.Garage.Instances[0]
return garage.Peer{
return garage.RemotePeer{
ID: inst.ID,
IP: thisHost.Nebula.IP,
RPCPort: inst.RPCPort,
S3APIPort: inst.S3APIPort,

5
entrypoint/src/bootstrap/hosts.go
Unescape View File

@ -22,8 +22,9 @@ type NebulaHost struct {
// GarageHost describes a single garage instance in the GarageHost.
type GarageHostInstance struct {
RPCPort int `yaml:"rpc_port"`
S3APIPort int `yaml:"s3_api_port"`
ID string `yaml:"id"`
RPCPort int `yaml:"rpc_port"`
S3APIPort int `yaml:"s3_api_port"`
}
// GarageHost describes the garage configuration of a Host which is relevant for

8
entrypoint/src/cmd/entrypoint/daemon_util.go
Unescape View File

@ -4,6 +4,7 @@ import (
"context"
"cryptic-net/bootstrap"
"cryptic-net/daemon"
"cryptic-net/garage"
"fmt"
"time"
)
@ -25,7 +26,14 @@ func mergeDaemonConfigIntoBootstrap(
host.Garage = new(bootstrap.GarageHost)
for _, alloc := range allocs {
id, err := garage.InitAlloc(alloc.MetaPath)
if err != nil {
return bootstrap.Bootstrap{}, fmt.Errorf("initializing alloc at %q: %w", alloc.MetaPath, err)
}
host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
ID: id,
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
})

66
entrypoint/src/cmd/entrypoint/garage_util.go
Unescape View File

@ -7,7 +7,6 @@ import (
"cryptic-net/garage"
"fmt"
"net"
"os"
"path/filepath"
"strconv"
@ -66,6 +65,25 @@ func waitForGarageAndNebula(
}
// bootstrapGarageHostForAlloc returns the bootstrap.GarageHostInstance which
// corresponds with the given alloc from the daemon config. This will panic if
// no associated instance can be found.
//
// This assumes that mergeDaemonConfigIntoBootstrap has already been called.
func bootstrapGarageHostForAlloc(
host bootstrap.Host,
alloc daemon.ConfigStorageAllocation,
) bootstrap.GarageHostInstance {
for _, inst := range host.Garage.Instances {
if inst.RPCPort == alloc.RPCPort {
return inst
}
}
panic(fmt.Sprintf("could not find alloc %+v in the bootstrap data", alloc))
}
func garageWriteChildConfig(
hostBootstrap bootstrap.Bootstrap,
alloc daemon.ConfigStorageAllocation,
@ -73,28 +91,17 @@ func garageWriteChildConfig(
string, error,
) {
if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
}
thisHost := hostBootstrap.ThisHost()
peer := garage.Peer{
IP: thisHost.Nebula.IP,
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
}
pubKey, privKey := peer.RPCPeerKey()
nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
id := bootstrapGarageHostForAlloc(thisHost, alloc).ID
peer := garage.LocalPeer{
RemotePeer: garage.RemotePeer{
ID: id,
IP: thisHost.Nebula.IP,
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
},
AdminPort: alloc.AdminPort,
}
garageTomlPath := filepath.Join(
@ -108,9 +115,9 @@ func garageWriteChildConfig(
RPCSecret: hostBootstrap.GarageRPCSecret,
AdminToken: hostBootstrap.GarageAdminToken,
RPCAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
APIAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
AdminAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.AdminPort)),
RPCAddr: peer.RPCAddr(),
S3APIAddr: peer.S3APIAddr(),
AdminAddr: peer.AdminAddr(),
BootstrapPeers: hostBootstrap.GarageRPCPeerAddrs(),
})
@ -224,7 +231,6 @@ func garageApplyLayout(
adminClient = newGarageAdminClient(hostBootstrap, daemonConfig)
thisHost = hostBootstrap.ThisHost()
hostName = thisHost.Name
ip = thisHost.Nebula.IP
allocs = daemonConfig.Storage.Allocations
)
@ -239,13 +245,9 @@ func garageApplyLayout(
for _, alloc := range allocs {
peer := garage.Peer{
IP: ip,
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
}
id := bootstrapGarageHostForAlloc(thisHost, alloc).ID
clusterLayout[peer.RPCPeerID()] = peerLayout{
clusterLayout[id] = peerLayout{
Capacity: alloc.Capacity / 100,
Zone: hostName,
Tags: []string{},

87
entrypoint/src/garage/garage.go
Unescape View File

@ -2,6 +2,15 @@
// setting up garage configs, processes, and deployments.
package garage
import (
"encoding/hex"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
)
const (
// Region is the region which garage is configured with.
@ -15,3 +24,81 @@ const (
// cluster. We currently only support a factor of 3.
ReplicationFactor = 3
)
func nodeKeyPath(metaDirPath string) string {
return filepath.Join(metaDirPath, "node_key")
}
func nodeKeyPubPath(metaDirPath string) string {
return filepath.Join(metaDirPath, "node_key.pub")
}
// LoadAllocID returns the peer ID (ie the public key) of the node at the given
// meta directory.
func LoadAllocID(metaDirPath string) (string, error) {
nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
pubKey, err := os.ReadFile(nodeKeyPubPath)
if err != nil {
return "", fmt.Errorf("reading %q: %w", nodeKeyPubPath, err)
}
return hex.EncodeToString(pubKey), nil
}
// InitAlloc initializes the meta directory and keys for a particular
// allocation, if it hasn't been done so already. It returns the peer ID (ie the
// public key) in any case.
func InitAlloc(metaDirPath string) (string, error) {
var err error
exists := func(path string) bool {
if err != nil {
return false
} else if _, err = os.Stat(path); errors.Is(err, fs.ErrNotExist) {
return false
} else if err != nil {
err = fmt.Errorf("checking if %q exists: %w", path, err)
return false
}
return true
}
nodeKeyPath := nodeKeyPath(metaDirPath)
nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
nodeKeyPathExists := exists(nodeKeyPath)
nodeKeyPubPathExists := exists(nodeKeyPubPath)
if err != nil {
return "", err
} else if nodeKeyPubPathExists != nodeKeyPathExists {
return "", fmt.Errorf("%q or %q exist without the other existing", nodeKeyPath, nodeKeyPubPath)
} else if nodeKeyPathExists {
return LoadAllocID(metaDirPath)
}
// node key hasn't been written, write it
if err := os.MkdirAll(metaDirPath, 0750); err != nil {
return "", fmt.Errorf("making directory %q: %w", metaDirPath, err)
}
pubKey, privKey := GeneratePeerKey()
if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
}
return "", nil
}

41
entrypoint/src/garage/infinite_reader.go
Unescape View File

@ -1,41 +0,0 @@
package garage
import "io"
type infiniteReader struct {
b []byte
i int
}
// NewInfiniteReader returns a reader which will produce the given bytes in
// repetition. len(b) must be greater than 0.
func NewInfiniteReader(b []byte) io.Reader {
if len(b) == 0 {
panic("len(b) must be greater than 0")
}
return &infiniteReader{b: b}
}
func (r *infiniteReader) Read(b []byte) (int, error) {
// here, have a puzzle
var n int
for {
n += copy(b[n:], r.b[r.i:])
if r.i > 0 {
n += copy(b[n:], r.b[:r.i])
}
r.i = (r.i + n) % len(r.b)
if n >= len(b) {
return n, nil
}
}
}

101
entrypoint/src/garage/infinite_reader_test.go
Unescape View File

@ -1,101 +0,0 @@
package garage
import (
"bytes"
"strconv"
"testing"
)
func TestInfiniteReader(t *testing.T) {
tests := []struct {
in []byte
size int
exp []string
}{
{
in: []byte("a"),
size: 1,
exp: []string{"a"},
},
{
in: []byte("ab"),
size: 1,
exp: []string{"a", "b"},
},
{
in: []byte("ab"),
size: 2,
exp: []string{"ab"},
},
{
in: []byte("ab"),
size: 3,
exp: []string{"aba", "bab"},
},
{
in: []byte("ab"),
size: 4,
exp: []string{"abab"},
},
{
in: []byte("ab"),
size: 5,
exp: []string{"ababa", "babab"},
},
{
in: []byte("abc"),
size: 1,
exp: []string{"a", "b", "c"},
},
{
in: []byte("abc"),
size: 2,
exp: []string{"ab", "ca", "bc"},
},
{
in: []byte("abc"),
size: 3,
exp: []string{"abc"},
},
{
in: []byte("abc"),
size: 4,
exp: []string{"abca", "bcab", "cabc"},
},
{
in: []byte("abc"),
size: 5,
exp: []string{"abcab", "cabca", "bcabc"},
},
}
for i, test := range tests {
t.Run(strconv.Itoa(i), func(t *testing.T) {
r := NewInfiniteReader(test.in)
buf := make([]byte, test.size)
assertRead := func(expBuf []byte) {
n, err := r.Read(buf)
if !bytes.Equal(buf, expBuf) {
t.Fatalf("expected bytes %q, got %q", expBuf, buf)
} else if n != len(buf) {
t.Fatalf("expected n %d, got %d", len(buf), n)
} else if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
for i := 0; i < 3; i++ {
for _, expStr := range test.exp {
assertRead([]byte(expStr))
}
}
})
}
}

55
entrypoint/src/garage/peer.go
Unescape View File

@ -2,35 +2,32 @@ package garage
import (
"crypto/ed25519"
"encoding/hex"
"crypto/rand"
"fmt"
"net"
"strconv"
)
// Peer describes all information necessary to connect to a given garage node.
type Peer struct {
// RemotePeer describes all information necessary to connect to a given garage
// node.
type RemotePeer struct {
ID string
IP string
RPCPort int
S3APIPort int
}
// RPCPeerKey deterministically generates a public/private keys which can
// be used as a garage node key.
//
// DANGER: This function will deterministically produce public/private keys
// given some arbitrary input. This is NEVER what you want. It's only being used
// in cryptic-net for a very specific purpose for which I think it's ok and is
// very necessary, and people are probably _still_ going to yell at me.
//
func (p Peer) RPCPeerKey() (pubKey, privKey []byte) {
input := []byte(net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort)))
// LocalPeer describes the configuration of a local garage instance.
type LocalPeer struct {
RemotePeer
// Append the length of the input to the input, so that the input "foo"
// doesn't generate the same key as the input "foofoo".
input = strconv.AppendInt(input, int64(len(input)), 10)
AdminPort int
}
pubKey, privKey, err := ed25519.GenerateKey(NewInfiniteReader(input))
// GeneratePeerKey generates and returns a public/private key pair for a garage
// instance.
func GeneratePeerKey() (pubKey, privKey []byte) {
pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
panic(err)
}
@ -38,29 +35,23 @@ func (p Peer) RPCPeerKey() (pubKey, privKey []byte) {
return pubKey, privKey
}
// RPCPeerID returns the peer ID of the garage node for use in communicating
// over RPC.
//
// DANGER: See warning on RPCPeerKey.
func (p Peer) RPCPeerID() string {
pubKey, _ := p.RPCPeerKey()
return hex.EncodeToString(pubKey)
}
// RPCAddr returns the address of the peer's RPC port.
func (p Peer) RPCAddr() string {
func (p RemotePeer) RPCAddr() string {
return net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort))
}
// RPCPeerAddr returns the full peer address (e.g. "id@ip:port") of the garage
// node for use in communicating over RPC.
//
// DANGER: See warning on RPCPeerKey.
func (p Peer) RPCPeerAddr() string {
return fmt.Sprintf("%s@%s", p.RPCPeerID(), p.RPCAddr())
func (p RemotePeer) RPCPeerAddr() string {
return fmt.Sprintf("%s@%s", p.ID, p.RPCAddr())
}
// S3APIAddr returns the address of the peer's S3 API port.
func (p Peer) S3APIAddr() string {
func (p RemotePeer) S3APIAddr() string {
return net.JoinHostPort(p.IP, strconv.Itoa(p.S3APIPort))
}
// AdminAddr returns the address of the peer's S3 API port.
func (p LocalPeer) AdminAddr() string {
return net.JoinHostPort(p.IP, strconv.Itoa(p.AdminPort))
}

4
entrypoint/src/garage/tpl.go
Unescape View File

@ -17,7 +17,7 @@ type GarageTomlData struct {
AdminToken string
RPCAddr string
APIAddr string
S3APIAddr string
AdminAddr string
BootstrapPeers []string
@ -39,7 +39,7 @@ bootstrap_peers = [{{- range .BootstrapPeers }}
{{ end -}}]
[s3_api]
api_bind_addr = "{{ .APIAddr }}"
api_bind_addr = "{{ .S3APIAddr }}"
s3_region = "garage"
[admin]

Write Preview
Loading…
Cancel
Save