diff --git a/entrypoint/src/bootstrap/garage.go b/entrypoint/src/bootstrap/garage.go index afb95f3..621a446 100644 --- a/entrypoint/src/bootstrap/garage.go +++ b/entrypoint/src/bootstrap/garage.go @@ -12,9 +12,9 @@ const ( ) // GaragePeers returns a Peer for each known garage instance in the network. -func (b Bootstrap) GaragePeers() []garage.Peer { +func (b Bootstrap) GaragePeers() []garage.RemotePeer { - var peers []garage.Peer + var peers []garage.RemotePeer for _, host := range b.Hosts { @@ -24,7 +24,8 @@ func (b Bootstrap) GaragePeers() []garage.Peer { for _, instance := range host.Garage.Instances { - peer := garage.Peer{ + peer := garage.RemotePeer{ + ID: instance.ID, IP: host.Nebula.IP, RPCPort: instance.RPCPort, S3APIPort: instance.S3APIPort, @@ -50,13 +51,14 @@ func (b Bootstrap) GarageRPCPeerAddrs() []string { // ChooseGaragePeer returns a Peer for a garage instance from the network. It // will prefer a garage instance on this particular host, if there is one, but // will otherwise return a random endpoint. -func (b Bootstrap) ChooseGaragePeer() garage.Peer { +func (b Bootstrap) ChooseGaragePeer() garage.RemotePeer { thisHost := b.ThisHost() if thisHost.Garage != nil && len(thisHost.Garage.Instances) > 0 { inst := thisHost.Garage.Instances[0] - return garage.Peer{ + return garage.RemotePeer{ + ID: inst.ID, IP: thisHost.Nebula.IP, RPCPort: inst.RPCPort, S3APIPort: inst.S3APIPort, diff --git a/entrypoint/src/bootstrap/hosts.go b/entrypoint/src/bootstrap/hosts.go index 0042457..c5fb37b 100644 --- a/entrypoint/src/bootstrap/hosts.go +++ b/entrypoint/src/bootstrap/hosts.go @@ -22,8 +22,9 @@ type NebulaHost struct { // GarageHost describes a single garage instance in the GarageHost. type GarageHostInstance struct { - RPCPort int `yaml:"rpc_port"` - S3APIPort int `yaml:"s3_api_port"` + ID string `yaml:"id"` + RPCPort int `yaml:"rpc_port"` + S3APIPort int `yaml:"s3_api_port"` } // GarageHost describes the garage configuration of a Host which is relevant for diff --git a/entrypoint/src/cmd/entrypoint/daemon_util.go b/entrypoint/src/cmd/entrypoint/daemon_util.go index 30f42c9..e66d2bb 100644 --- a/entrypoint/src/cmd/entrypoint/daemon_util.go +++ b/entrypoint/src/cmd/entrypoint/daemon_util.go @@ -4,6 +4,7 @@ import ( "context" "cryptic-net/bootstrap" "cryptic-net/daemon" + "cryptic-net/garage" "fmt" "time" ) @@ -25,7 +26,14 @@ func mergeDaemonConfigIntoBootstrap( host.Garage = new(bootstrap.GarageHost) for _, alloc := range allocs { + + id, err := garage.InitAlloc(alloc.MetaPath) + if err != nil { + return bootstrap.Bootstrap{}, fmt.Errorf("initializing alloc at %q: %w", alloc.MetaPath, err) + } + host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{ + ID: id, RPCPort: alloc.RPCPort, S3APIPort: alloc.S3APIPort, }) diff --git a/entrypoint/src/cmd/entrypoint/garage_util.go b/entrypoint/src/cmd/entrypoint/garage_util.go index 5d44e01..ec88cca 100644 --- a/entrypoint/src/cmd/entrypoint/garage_util.go +++ b/entrypoint/src/cmd/entrypoint/garage_util.go @@ -7,7 +7,6 @@ import ( "cryptic-net/garage" "fmt" "net" - "os" "path/filepath" "strconv" @@ -66,6 +65,25 @@ func waitForGarageAndNebula( } +// bootstrapGarageHostForAlloc returns the bootstrap.GarageHostInstance which +// corresponds with the given alloc from the daemon config. This will panic if +// no associated instance can be found. +// +// This assumes that mergeDaemonConfigIntoBootstrap has already been called. +func bootstrapGarageHostForAlloc( + host bootstrap.Host, + alloc daemon.ConfigStorageAllocation, +) bootstrap.GarageHostInstance { + + for _, inst := range host.Garage.Instances { + if inst.RPCPort == alloc.RPCPort { + return inst + } + } + + panic(fmt.Sprintf("could not find alloc %+v in the bootstrap data", alloc)) +} + func garageWriteChildConfig( hostBootstrap bootstrap.Bootstrap, alloc daemon.ConfigStorageAllocation, @@ -73,28 +91,17 @@ func garageWriteChildConfig( string, error, ) { - if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil { - return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err) - } - thisHost := hostBootstrap.ThisHost() + id := bootstrapGarageHostForAlloc(thisHost, alloc).ID - peer := garage.Peer{ - IP: thisHost.Nebula.IP, - RPCPort: alloc.RPCPort, - S3APIPort: alloc.S3APIPort, - } - - pubKey, privKey := peer.RPCPeerKey() - - nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key") - nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub") - - if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil { - return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err) - - } else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil { - return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err) + peer := garage.LocalPeer{ + RemotePeer: garage.RemotePeer{ + ID: id, + IP: thisHost.Nebula.IP, + RPCPort: alloc.RPCPort, + S3APIPort: alloc.S3APIPort, + }, + AdminPort: alloc.AdminPort, } garageTomlPath := filepath.Join( @@ -108,9 +115,9 @@ func garageWriteChildConfig( RPCSecret: hostBootstrap.GarageRPCSecret, AdminToken: hostBootstrap.GarageAdminToken, - RPCAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)), - APIAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)), - AdminAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.AdminPort)), + RPCAddr: peer.RPCAddr(), + S3APIAddr: peer.S3APIAddr(), + AdminAddr: peer.AdminAddr(), BootstrapPeers: hostBootstrap.GarageRPCPeerAddrs(), }) @@ -224,7 +231,6 @@ func garageApplyLayout( adminClient = newGarageAdminClient(hostBootstrap, daemonConfig) thisHost = hostBootstrap.ThisHost() hostName = thisHost.Name - ip = thisHost.Nebula.IP allocs = daemonConfig.Storage.Allocations ) @@ -239,13 +245,9 @@ func garageApplyLayout( for _, alloc := range allocs { - peer := garage.Peer{ - IP: ip, - RPCPort: alloc.RPCPort, - S3APIPort: alloc.S3APIPort, - } + id := bootstrapGarageHostForAlloc(thisHost, alloc).ID - clusterLayout[peer.RPCPeerID()] = peerLayout{ + clusterLayout[id] = peerLayout{ Capacity: alloc.Capacity / 100, Zone: hostName, Tags: []string{}, diff --git a/entrypoint/src/garage/garage.go b/entrypoint/src/garage/garage.go index 3bb4cfb..0eb0146 100644 --- a/entrypoint/src/garage/garage.go +++ b/entrypoint/src/garage/garage.go @@ -2,6 +2,15 @@ // setting up garage configs, processes, and deployments. package garage +import ( + "encoding/hex" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" +) + const ( // Region is the region which garage is configured with. @@ -15,3 +24,81 @@ const ( // cluster. We currently only support a factor of 3. ReplicationFactor = 3 ) + +func nodeKeyPath(metaDirPath string) string { + return filepath.Join(metaDirPath, "node_key") +} + +func nodeKeyPubPath(metaDirPath string) string { + return filepath.Join(metaDirPath, "node_key.pub") +} + +// LoadAllocID returns the peer ID (ie the public key) of the node at the given +// meta directory. +func LoadAllocID(metaDirPath string) (string, error) { + nodeKeyPubPath := nodeKeyPubPath(metaDirPath) + + pubKey, err := os.ReadFile(nodeKeyPubPath) + if err != nil { + return "", fmt.Errorf("reading %q: %w", nodeKeyPubPath, err) + } + + return hex.EncodeToString(pubKey), nil +} + +// InitAlloc initializes the meta directory and keys for a particular +// allocation, if it hasn't been done so already. It returns the peer ID (ie the +// public key) in any case. +func InitAlloc(metaDirPath string) (string, error) { + + var err error + + exists := func(path string) bool { + + if err != nil { + return false + + } else if _, err = os.Stat(path); errors.Is(err, fs.ErrNotExist) { + return false + + } else if err != nil { + err = fmt.Errorf("checking if %q exists: %w", path, err) + return false + } + + return true + } + + nodeKeyPath := nodeKeyPath(metaDirPath) + nodeKeyPubPath := nodeKeyPubPath(metaDirPath) + + nodeKeyPathExists := exists(nodeKeyPath) + nodeKeyPubPathExists := exists(nodeKeyPubPath) + + if err != nil { + return "", err + + } else if nodeKeyPubPathExists != nodeKeyPathExists { + return "", fmt.Errorf("%q or %q exist without the other existing", nodeKeyPath, nodeKeyPubPath) + + } else if nodeKeyPathExists { + return LoadAllocID(metaDirPath) + } + + // node key hasn't been written, write it + + if err := os.MkdirAll(metaDirPath, 0750); err != nil { + return "", fmt.Errorf("making directory %q: %w", metaDirPath, err) + } + + pubKey, privKey := GeneratePeerKey() + + if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil { + return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err) + + } else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil { + return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err) + } + + return "", nil +} diff --git a/entrypoint/src/garage/infinite_reader.go b/entrypoint/src/garage/infinite_reader.go deleted file mode 100644 index 1b137f7..0000000 --- a/entrypoint/src/garage/infinite_reader.go +++ /dev/null @@ -1,41 +0,0 @@ -package garage - -import "io" - -type infiniteReader struct { - b []byte - i int -} - -// NewInfiniteReader returns a reader which will produce the given bytes in -// repetition. len(b) must be greater than 0. -func NewInfiniteReader(b []byte) io.Reader { - - if len(b) == 0 { - panic("len(b) must be greater than 0") - } - - return &infiniteReader{b: b} -} - -func (r *infiniteReader) Read(b []byte) (int, error) { - - // here, have a puzzle - - var n int - - for { - - n += copy(b[n:], r.b[r.i:]) - - if r.i > 0 { - n += copy(b[n:], r.b[:r.i]) - } - - r.i = (r.i + n) % len(r.b) - - if n >= len(b) { - return n, nil - } - } -} diff --git a/entrypoint/src/garage/infinite_reader_test.go b/entrypoint/src/garage/infinite_reader_test.go deleted file mode 100644 index 477d7e9..0000000 --- a/entrypoint/src/garage/infinite_reader_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package garage - -import ( - "bytes" - "strconv" - "testing" -) - -func TestInfiniteReader(t *testing.T) { - - tests := []struct { - in []byte - size int - exp []string - }{ - { - in: []byte("a"), - size: 1, - exp: []string{"a"}, - }, - { - in: []byte("ab"), - size: 1, - exp: []string{"a", "b"}, - }, - { - in: []byte("ab"), - size: 2, - exp: []string{"ab"}, - }, - { - in: []byte("ab"), - size: 3, - exp: []string{"aba", "bab"}, - }, - { - in: []byte("ab"), - size: 4, - exp: []string{"abab"}, - }, - { - in: []byte("ab"), - size: 5, - exp: []string{"ababa", "babab"}, - }, - { - in: []byte("abc"), - size: 1, - exp: []string{"a", "b", "c"}, - }, - { - in: []byte("abc"), - size: 2, - exp: []string{"ab", "ca", "bc"}, - }, - { - in: []byte("abc"), - size: 3, - exp: []string{"abc"}, - }, - { - in: []byte("abc"), - size: 4, - exp: []string{"abca", "bcab", "cabc"}, - }, - { - in: []byte("abc"), - size: 5, - exp: []string{"abcab", "cabca", "bcabc"}, - }, - } - - for i, test := range tests { - t.Run(strconv.Itoa(i), func(t *testing.T) { - - r := NewInfiniteReader(test.in) - buf := make([]byte, test.size) - - assertRead := func(expBuf []byte) { - - n, err := r.Read(buf) - - if !bytes.Equal(buf, expBuf) { - t.Fatalf("expected bytes %q, got %q", expBuf, buf) - - } else if n != len(buf) { - t.Fatalf("expected n %d, got %d", len(buf), n) - - } else if err != nil { - t.Fatalf("unexpected error: %v", err) - } - } - - for i := 0; i < 3; i++ { - for _, expStr := range test.exp { - assertRead([]byte(expStr)) - } - } - }) - } -} diff --git a/entrypoint/src/garage/peer.go b/entrypoint/src/garage/peer.go index 17b671a..f2fa8cd 100644 --- a/entrypoint/src/garage/peer.go +++ b/entrypoint/src/garage/peer.go @@ -2,35 +2,32 @@ package garage import ( "crypto/ed25519" - "encoding/hex" + "crypto/rand" "fmt" "net" "strconv" ) -// Peer describes all information necessary to connect to a given garage node. -type Peer struct { +// RemotePeer describes all information necessary to connect to a given garage +// node. +type RemotePeer struct { + ID string IP string RPCPort int S3APIPort int } -// RPCPeerKey deterministically generates a public/private keys which can -// be used as a garage node key. -// -// DANGER: This function will deterministically produce public/private keys -// given some arbitrary input. This is NEVER what you want. It's only being used -// in cryptic-net for a very specific purpose for which I think it's ok and is -// very necessary, and people are probably _still_ going to yell at me. -// -func (p Peer) RPCPeerKey() (pubKey, privKey []byte) { - input := []byte(net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort))) +// LocalPeer describes the configuration of a local garage instance. +type LocalPeer struct { + RemotePeer - // Append the length of the input to the input, so that the input "foo" - // doesn't generate the same key as the input "foofoo". - input = strconv.AppendInt(input, int64(len(input)), 10) + AdminPort int +} - pubKey, privKey, err := ed25519.GenerateKey(NewInfiniteReader(input)) +// GeneratePeerKey generates and returns a public/private key pair for a garage +// instance. +func GeneratePeerKey() (pubKey, privKey []byte) { + pubKey, privKey, err := ed25519.GenerateKey(rand.Reader) if err != nil { panic(err) } @@ -38,29 +35,23 @@ func (p Peer) RPCPeerKey() (pubKey, privKey []byte) { return pubKey, privKey } -// RPCPeerID returns the peer ID of the garage node for use in communicating -// over RPC. -// -// DANGER: See warning on RPCPeerKey. -func (p Peer) RPCPeerID() string { - pubKey, _ := p.RPCPeerKey() - return hex.EncodeToString(pubKey) -} - // RPCAddr returns the address of the peer's RPC port. -func (p Peer) RPCAddr() string { +func (p RemotePeer) RPCAddr() string { return net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort)) } // RPCPeerAddr returns the full peer address (e.g. "id@ip:port") of the garage // node for use in communicating over RPC. -// -// DANGER: See warning on RPCPeerKey. -func (p Peer) RPCPeerAddr() string { - return fmt.Sprintf("%s@%s", p.RPCPeerID(), p.RPCAddr()) +func (p RemotePeer) RPCPeerAddr() string { + return fmt.Sprintf("%s@%s", p.ID, p.RPCAddr()) } // S3APIAddr returns the address of the peer's S3 API port. -func (p Peer) S3APIAddr() string { +func (p RemotePeer) S3APIAddr() string { return net.JoinHostPort(p.IP, strconv.Itoa(p.S3APIPort)) } + +// AdminAddr returns the address of the peer's S3 API port. +func (p LocalPeer) AdminAddr() string { + return net.JoinHostPort(p.IP, strconv.Itoa(p.AdminPort)) +} diff --git a/entrypoint/src/garage/tpl.go b/entrypoint/src/garage/tpl.go index 351f03c..e0dedc8 100644 --- a/entrypoint/src/garage/tpl.go +++ b/entrypoint/src/garage/tpl.go @@ -17,7 +17,7 @@ type GarageTomlData struct { AdminToken string RPCAddr string - APIAddr string + S3APIAddr string AdminAddr string BootstrapPeers []string @@ -39,7 +39,7 @@ bootstrap_peers = [{{- range .BootstrapPeers }} {{ end -}}] [s3_api] -api_bind_addr = "{{ .APIAddr }}" +api_bind_addr = "{{ .S3APIAddr }}" s3_region = "garage" [admin]