Use a real private key for garage instances

entrypoint/src/bootstrap/garage.go

@@ -12,9 +12,9 @@ const (
 )
 
 // GaragePeers returns a Peer for each known garage instance in the network.
-func (b Bootstrap) GaragePeers() []garage.Peer {
+func (b Bootstrap) GaragePeers() []garage.RemotePeer {
 
-	var peers []garage.Peer
+	var peers []garage.RemotePeer
 
 	for _, host := range b.Hosts {
 
@@ -24,7 +24,8 @@ func (b Bootstrap) GaragePeers() []garage.Peer {
 
 		for _, instance := range host.Garage.Instances {
 
-			peer := garage.Peer{
+			peer := garage.RemotePeer{
+				ID:        instance.ID,
 				IP:        host.Nebula.IP,
 				RPCPort:   instance.RPCPort,
 				S3APIPort: instance.S3APIPort,
@@ -50,13 +51,14 @@ func (b Bootstrap) GarageRPCPeerAddrs() []string {
 // ChooseGaragePeer returns a Peer for a garage instance from the network. It
 // will prefer a garage instance on this particular host, if there is one, but
 // will otherwise return a random endpoint.
-func (b Bootstrap) ChooseGaragePeer() garage.Peer {
+func (b Bootstrap) ChooseGaragePeer() garage.RemotePeer {
 
 	thisHost := b.ThisHost()
 
 	if thisHost.Garage != nil && len(thisHost.Garage.Instances) > 0 {
 		inst := thisHost.Garage.Instances[0]
-		return garage.Peer{
+		return garage.RemotePeer{
+			ID:        inst.ID,
 			IP:        thisHost.Nebula.IP,
 			RPCPort:   inst.RPCPort,
 			S3APIPort: inst.S3APIPort,

entrypoint/src/bootstrap/hosts.go

@@ -22,8 +22,9 @@ type NebulaHost struct {
 
 // GarageHost describes a single garage instance in the GarageHost.
 type GarageHostInstance struct {
-	RPCPort   int `yaml:"rpc_port"`
+	ID        string `yaml:"id"`
-	S3APIPort int `yaml:"s3_api_port"`
+	RPCPort   int    `yaml:"rpc_port"`
+	S3APIPort int    `yaml:"s3_api_port"`
 }
 
 // GarageHost describes the garage configuration of a Host which is relevant for

entrypoint/src/cmd/entrypoint/daemon_util.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"cryptic-net/bootstrap"
 	"cryptic-net/daemon"
+	"cryptic-net/garage"
 	"fmt"
 	"time"
 )
@@ -25,7 +26,14 @@ func mergeDaemonConfigIntoBootstrap(
 		host.Garage = new(bootstrap.GarageHost)
 
 		for _, alloc := range allocs {
+
+			id, err := garage.InitAlloc(alloc.MetaPath)
+			if err != nil {
+				return bootstrap.Bootstrap{}, fmt.Errorf("initializing alloc at %q: %w", alloc.MetaPath, err)
+			}
+
 			host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
+				ID:        id,
 				RPCPort:   alloc.RPCPort,
 				S3APIPort: alloc.S3APIPort,
 			})

entrypoint/src/cmd/entrypoint/garage_util.go

@@ -7,7 +7,6 @@ import (
 	"cryptic-net/garage"
 	"fmt"
 	"net"
-	"os"
 	"path/filepath"
 	"strconv"
 
@@ -66,6 +65,25 @@ func waitForGarageAndNebula(
 
 }
 
+// bootstrapGarageHostForAlloc returns the bootstrap.GarageHostInstance which
+// corresponds with the given alloc from the daemon config. This will panic if
+// no associated instance can be found.
+//
+// This assumes that mergeDaemonConfigIntoBootstrap has already been called.
+func bootstrapGarageHostForAlloc(
+	host bootstrap.Host,
+	alloc daemon.ConfigStorageAllocation,
+) bootstrap.GarageHostInstance {
+
+	for _, inst := range host.Garage.Instances {
+		if inst.RPCPort == alloc.RPCPort {
+			return inst
+		}
+	}
+
+	panic(fmt.Sprintf("could not find alloc %+v in the bootstrap data", alloc))
+}
+
 func garageWriteChildConfig(
 	hostBootstrap bootstrap.Bootstrap,
 	alloc daemon.ConfigStorageAllocation,
@@ -73,28 +91,17 @@ func garageWriteChildConfig(
 	string, error,
 ) {
 
-	if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
-		return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
-	}
-
 	thisHost := hostBootstrap.ThisHost()
+	id := bootstrapGarageHostForAlloc(thisHost, alloc).ID
 
-	peer := garage.Peer{
+	peer := garage.LocalPeer{
-		IP:        thisHost.Nebula.IP,
+		RemotePeer: garage.RemotePeer{
-		RPCPort:   alloc.RPCPort,
+			ID:        id,
-		S3APIPort: alloc.S3APIPort,
+			IP:        thisHost.Nebula.IP,
-	}
+			RPCPort:   alloc.RPCPort,
-
+			S3APIPort: alloc.S3APIPort,
-	pubKey, privKey := peer.RPCPeerKey()
+		},
-
+		AdminPort: alloc.AdminPort,
-	nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
-	nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
-
-	if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
-		return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
-
-	} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
-		return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
 	}
 
 	garageTomlPath := filepath.Join(
@@ -108,9 +115,9 @@ func garageWriteChildConfig(
 		RPCSecret:  hostBootstrap.GarageRPCSecret,
 		AdminToken: hostBootstrap.GarageAdminToken,
 
-		RPCAddr:   net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
+		RPCAddr:   peer.RPCAddr(),
-		APIAddr:   net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
+		S3APIAddr: peer.S3APIAddr(),
-		AdminAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.AdminPort)),
+		AdminAddr: peer.AdminAddr(),
 
 		BootstrapPeers: hostBootstrap.GarageRPCPeerAddrs(),
 	})
@@ -224,7 +231,6 @@ func garageApplyLayout(
 		adminClient = newGarageAdminClient(hostBootstrap, daemonConfig)
 		thisHost    = hostBootstrap.ThisHost()
 		hostName    = thisHost.Name
-		ip          = thisHost.Nebula.IP
 		allocs      = daemonConfig.Storage.Allocations
 	)
 
@@ -239,13 +245,9 @@ func garageApplyLayout(
 
 		for _, alloc := range allocs {
 
-			peer := garage.Peer{
+			id := bootstrapGarageHostForAlloc(thisHost, alloc).ID
-				IP:        ip,
-				RPCPort:   alloc.RPCPort,
-				S3APIPort: alloc.S3APIPort,
-			}
 
-			clusterLayout[peer.RPCPeerID()] = peerLayout{
+			clusterLayout[id] = peerLayout{
 				Capacity: alloc.Capacity / 100,
 				Zone:     hostName,
 				Tags:     []string{},

entrypoint/src/garage/garage.go

@@ -2,6 +2,15 @@
 // setting up garage configs, processes, and deployments.
 package garage
 
+import (
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
 const (
 
 	// Region is the region which garage is configured with.
@@ -15,3 +24,81 @@ const (
 	// cluster. We currently only support a factor of 3.
 	ReplicationFactor = 3
 )
+
+func nodeKeyPath(metaDirPath string) string {
+	return filepath.Join(metaDirPath, "node_key")
+}
+
+func nodeKeyPubPath(metaDirPath string) string {
+	return filepath.Join(metaDirPath, "node_key.pub")
+}
+
+// LoadAllocID returns the peer ID (ie the public key) of the node at the given
+// meta directory.
+func LoadAllocID(metaDirPath string) (string, error) {
+	nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
+
+	pubKey, err := os.ReadFile(nodeKeyPubPath)
+	if err != nil {
+		return "", fmt.Errorf("reading %q: %w", nodeKeyPubPath, err)
+	}
+
+	return hex.EncodeToString(pubKey), nil
+}
+
+// InitAlloc initializes the meta directory and keys for a particular
+// allocation, if it hasn't been done so already. It returns the peer ID (ie the
+// public key) in any case.
+func InitAlloc(metaDirPath string) (string, error) {
+
+	var err error
+
+	exists := func(path string) bool {
+
+		if err != nil {
+			return false
+
+		} else if _, err = os.Stat(path); errors.Is(err, fs.ErrNotExist) {
+			return false
+
+		} else if err != nil {
+			err = fmt.Errorf("checking if %q exists: %w", path, err)
+			return false
+		}
+
+		return true
+	}
+
+	nodeKeyPath := nodeKeyPath(metaDirPath)
+	nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
+
+	nodeKeyPathExists := exists(nodeKeyPath)
+	nodeKeyPubPathExists := exists(nodeKeyPubPath)
+
+	if err != nil {
+		return "", err
+
+	} else if nodeKeyPubPathExists != nodeKeyPathExists {
+		return "", fmt.Errorf("%q or %q exist without the other existing", nodeKeyPath, nodeKeyPubPath)
+
+	} else if nodeKeyPathExists {
+		return LoadAllocID(metaDirPath)
+	}
+
+	// node key hasn't been written, write it
+
+	if err := os.MkdirAll(metaDirPath, 0750); err != nil {
+		return "", fmt.Errorf("making directory %q: %w", metaDirPath, err)
+	}
+
+	pubKey, privKey := GeneratePeerKey()
+
+	if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
+		return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
+
+	} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
+		return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
+	}
+
+	return "", nil
+}

entrypoint/src/garage/infinite_reader.go

@@ -1,41 +0,0 @@
-package garage
-
-import "io"
-
-type infiniteReader struct {
-	b []byte
-	i int
-}
-
-// NewInfiniteReader returns a reader which will produce the given bytes in
-// repetition. len(b) must be greater than 0.
-func NewInfiniteReader(b []byte) io.Reader {
-
-	if len(b) == 0 {
-		panic("len(b) must be greater than 0")
-	}
-
-	return &infiniteReader{b: b}
-}
-
-func (r *infiniteReader) Read(b []byte) (int, error) {
-
-	// here, have a puzzle
-
-	var n int
-
-	for {
-
-		n += copy(b[n:], r.b[r.i:])
-
-		if r.i > 0 {
-			n += copy(b[n:], r.b[:r.i])
-		}
-
-		r.i = (r.i + n) % len(r.b)
-
-		if n >= len(b) {
-			return n, nil
-		}
-	}
-}

entrypoint/src/garage/infinite_reader_test.go

@@ -1,101 +0,0 @@
-package garage
-
-import (
-	"bytes"
-	"strconv"
-	"testing"
-)
-
-func TestInfiniteReader(t *testing.T) {
-
-	tests := []struct {
-		in   []byte
-		size int
-		exp  []string
-	}{
-		{
-			in:   []byte("a"),
-			size: 1,
-			exp:  []string{"a"},
-		},
-		{
-			in:   []byte("ab"),
-			size: 1,
-			exp:  []string{"a", "b"},
-		},
-		{
-			in:   []byte("ab"),
-			size: 2,
-			exp:  []string{"ab"},
-		},
-		{
-			in:   []byte("ab"),
-			size: 3,
-			exp:  []string{"aba", "bab"},
-		},
-		{
-			in:   []byte("ab"),
-			size: 4,
-			exp:  []string{"abab"},
-		},
-		{
-			in:   []byte("ab"),
-			size: 5,
-			exp:  []string{"ababa", "babab"},
-		},
-		{
-			in:   []byte("abc"),
-			size: 1,
-			exp:  []string{"a", "b", "c"},
-		},
-		{
-			in:   []byte("abc"),
-			size: 2,
-			exp:  []string{"ab", "ca", "bc"},
-		},
-		{
-			in:   []byte("abc"),
-			size: 3,
-			exp:  []string{"abc"},
-		},
-		{
-			in:   []byte("abc"),
-			size: 4,
-			exp:  []string{"abca", "bcab", "cabc"},
-		},
-		{
-			in:   []byte("abc"),
-			size: 5,
-			exp:  []string{"abcab", "cabca", "bcabc"},
-		},
-	}
-
-	for i, test := range tests {
-		t.Run(strconv.Itoa(i), func(t *testing.T) {
-
-			r := NewInfiniteReader(test.in)
-			buf := make([]byte, test.size)
-
-			assertRead := func(expBuf []byte) {
-
-				n, err := r.Read(buf)
-
-				if !bytes.Equal(buf, expBuf) {
-					t.Fatalf("expected bytes %q, got %q", expBuf, buf)
-
-				} else if n != len(buf) {
-					t.Fatalf("expected n %d, got %d", len(buf), n)
-
-				} else if err != nil {
-					t.Fatalf("unexpected error: %v", err)
-				}
-			}
-
-			for i := 0; i < 3; i++ {
-				for _, expStr := range test.exp {
-					assertRead([]byte(expStr))
-				}
-			}
-		})
-	}
-}

entrypoint/src/garage/peer.go

@@ -2,35 +2,32 @@ package garage
 
 import (
 	"crypto/ed25519"
-	"encoding/hex"
+	"crypto/rand"
 	"fmt"
 	"net"
 	"strconv"
 )
 
-// Peer describes all information necessary to connect to a given garage node.
+// RemotePeer describes all information necessary to connect to a given garage
-type Peer struct {
+// node.
+type RemotePeer struct {
+	ID        string
 	IP        string
 	RPCPort   int
 	S3APIPort int
 }
 
-// RPCPeerKey deterministically generates a public/private keys which can
+// LocalPeer describes the configuration of a local garage instance.
-// be used as a garage node key.
+type LocalPeer struct {
-//
+	RemotePeer
-// DANGER: This function will deterministically produce public/private keys
-// given some arbitrary input. This is NEVER what you want. It's only being used
-// in cryptic-net for a very specific purpose for which I think it's ok and is
-// very necessary, and people are probably _still_ going to yell at me.
-//
-func (p Peer) RPCPeerKey() (pubKey, privKey []byte) {
-	input := []byte(net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort)))
 
-	// Append the length of the input to the input, so that the input "foo"
+	AdminPort int
-	// doesn't generate the same key as the input "foofoo".
+}
-	input = strconv.AppendInt(input, int64(len(input)), 10)
 
-	pubKey, privKey, err := ed25519.GenerateKey(NewInfiniteReader(input))
+// GeneratePeerKey generates and returns a public/private key pair for a garage
+// instance.
+func GeneratePeerKey() (pubKey, privKey []byte) {
+	pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
 	if err != nil {
 		panic(err)
 	}
@@ -38,29 +35,23 @@ func (p Peer) RPCPeerKey() (pubKey, privKey []byte) {
 	return pubKey, privKey
 }
 
-// RPCPeerID returns the peer ID of the garage node for use in communicating
-// over RPC.
-//
-// DANGER: See warning on RPCPeerKey.
-func (p Peer) RPCPeerID() string {
-	pubKey, _ := p.RPCPeerKey()
-	return hex.EncodeToString(pubKey)
-}
-
 // RPCAddr returns the address of the peer's RPC port.
-func (p Peer) RPCAddr() string {
+func (p RemotePeer) RPCAddr() string {
 	return net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort))
 }
 
 // RPCPeerAddr returns the full peer address (e.g. "id@ip:port") of the garage
 // node for use in communicating over RPC.
-//
+func (p RemotePeer) RPCPeerAddr() string {
-// DANGER: See warning on RPCPeerKey.
+	return fmt.Sprintf("%s@%s", p.ID, p.RPCAddr())
-func (p Peer) RPCPeerAddr() string {
-	return fmt.Sprintf("%s@%s", p.RPCPeerID(), p.RPCAddr())
 }
 
 // S3APIAddr returns the address of the peer's S3 API port.
-func (p Peer) S3APIAddr() string {
+func (p RemotePeer) S3APIAddr() string {
 	return net.JoinHostPort(p.IP, strconv.Itoa(p.S3APIPort))
 }
+
+// AdminAddr returns the address of the peer's S3 API port.
+func (p LocalPeer) AdminAddr() string {
+	return net.JoinHostPort(p.IP, strconv.Itoa(p.AdminPort))
+}

entrypoint/src/garage/tpl.go

@@ -17,7 +17,7 @@ type GarageTomlData struct {
 	AdminToken string
 
 	RPCAddr   string
-	APIAddr   string
+	S3APIAddr string
 	AdminAddr string
 
 	BootstrapPeers []string
@@ -39,7 +39,7 @@ bootstrap_peers = [{{- range .BootstrapPeers }}
 {{ end -}}]
 
 [s3_api]
-api_bind_addr = "{{ .APIAddr }}"
+api_bind_addr = "{{ .S3APIAddr }}"
 s3_region = "garage"
 
 [admin]