From 842c169169a41f0917f48433b80780b268720b16 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Wed, 12 Jun 2024 10:18:33 +0200 Subject: [PATCH] Separate garage server logic into its own package --- go/bootstrap/garage.go | 11 --- go/cmd/entrypoint/daemon_util.go | 4 +- go/cmd/entrypoint/garage_util.go | 10 +- go/garage/client.go | 10 -- go/garage/garage.go | 140 +--------------------------- go/garage/garagesrv/garagesrv.go | 152 +++++++++++++++++++++++++++++++ go/garage/{ => garagesrv}/tpl.go | 16 ++-- go/garage/peer.go | 13 --- 8 files changed, 168 insertions(+), 188 deletions(-) create mode 100644 go/garage/garagesrv/garagesrv.go rename go/garage/{ => garagesrv}/tpl.go (87%) diff --git a/go/bootstrap/garage.go b/go/bootstrap/garage.go index 94f6cd5..e09c7ae 100644 --- a/go/bootstrap/garage.go +++ b/go/bootstrap/garage.go @@ -27,17 +27,6 @@ func (b Bootstrap) GaragePeers() []garage.RemotePeer { return peers } -// GarageRPCPeerAddrs returns the full RPC peer address for each known garage -// instance in the network. -func (b Bootstrap) GarageRPCPeerAddrs() []string { - var addrs []string - for _, peer := range b.GaragePeers() { - addr := peer.RPCPeerAddr() - addrs = append(addrs, addr) - } - return addrs -} - // ChooseGaragePeer returns a Peer for a garage instance from the network. It // will prefer a garage instance on this particular host, if there is one, but // will otherwise return a random endpoint. diff --git a/go/cmd/entrypoint/daemon_util.go b/go/cmd/entrypoint/daemon_util.go index c953f00..a85e8b4 100644 --- a/go/cmd/entrypoint/daemon_util.go +++ b/go/cmd/entrypoint/daemon_util.go @@ -5,7 +5,7 @@ import ( "fmt" "isle/bootstrap" "isle/daemon" - "isle/garage" + "isle/garage/garagesrv" "time" ) @@ -29,7 +29,7 @@ func coalesceDaemonConfigAndBootstrap( for i, alloc := range allocs { - id, rpcPort, err := garage.InitAlloc(alloc.MetaPath, alloc.RPCPort) + id, rpcPort, err := garagesrv.InitAlloc(alloc.MetaPath, alloc.RPCPort) if err != nil { return bootstrap.Bootstrap{}, daemon.Config{}, fmt.Errorf("initializing alloc at %q: %w", alloc.MetaPath, err) } diff --git a/go/cmd/entrypoint/garage_util.go b/go/cmd/entrypoint/garage_util.go index dff0b02..792c1d4 100644 --- a/go/cmd/entrypoint/garage_util.go +++ b/go/cmd/entrypoint/garage_util.go @@ -6,6 +6,7 @@ import ( "isle/bootstrap" "isle/daemon" "isle/garage" + "isle/garage/garagesrv" "net" "path/filepath" "strconv" @@ -128,18 +129,15 @@ func garageWriteChildConfig( envRuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort), ) - err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{ + err := garagesrv.WriteGarageTomlFile(garageTomlPath, garagesrv.GarageTomlData{ MetaPath: alloc.MetaPath, DataPath: alloc.DataPath, RPCSecret: hostBootstrap.Garage.RPCSecret, AdminToken: hostBootstrap.Garage.AdminToken, - RPCAddr: peer.RPCAddr(), - S3APIAddr: peer.S3APIAddr(), - AdminAddr: peer.AdminAddr(), - - BootstrapPeers: hostBootstrap.GarageRPCPeerAddrs(), + LocalPeer: peer, + BootstrapPeers: hostBootstrap.GaragePeers(), }) if err != nil { diff --git a/go/garage/client.go b/go/garage/client.go index f5c7b86..40c8456 100644 --- a/go/garage/client.go +++ b/go/garage/client.go @@ -1,8 +1,6 @@ package garage import ( - "crypto/rand" - "encoding/hex" "errors" "fmt" @@ -10,14 +8,6 @@ import ( "github.com/minio/minio-go/v7/pkg/credentials" ) -func randStr(l int) string { - b := make([]byte, l) - if _, err := rand.Read(b); err != nil { - panic(err) - } - return hex.EncodeToString(b) -} - // IsKeyNotFound returns true if the given error is the result of a key not // being found in a bucket. func IsKeyNotFound(err error) bool { diff --git a/go/garage/garage.go b/go/garage/garage.go index 8df9129..65095a0 100644 --- a/go/garage/garage.go +++ b/go/garage/garage.go @@ -1,17 +1,7 @@ -// Package garage contains helper functions and types which are useful for -// setting up garage configs, processes, and deployments. +// Package garage contains types and helpers related to interacting with garage +// processes via garage's APIs. package garage -import ( - "encoding/hex" - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "strconv" -) - const ( // Region is the region which garage is configured with. @@ -25,129 +15,3 @@ const ( // cluster. We currently only support a factor of 3. ReplicationFactor = 3 ) - -func nodeKeyPath(metaDirPath string) string { - return filepath.Join(metaDirPath, "node_key") -} - -func nodeKeyPubPath(metaDirPath string) string { - return filepath.Join(metaDirPath, "node_key.pub") -} - -func nodeRPCPortPath(metaDirPath string) string { - return filepath.Join(metaDirPath, "isle", "rpc_port") -} - -// loadAllocID returns the peer ID (ie the public key) of the node at the given -// meta directory. -func loadAllocID(metaDirPath string) (string, error) { - nodeKeyPubPath := nodeKeyPubPath(metaDirPath) - - pubKey, err := os.ReadFile(nodeKeyPubPath) - if err != nil { - return "", fmt.Errorf("reading %q: %w", nodeKeyPubPath, err) - } - - return hex.EncodeToString(pubKey), nil -} - -// InitAlloc initializes the meta directory and keys for a particular -// allocation, if it hasn't been done so already. It returns the peer ID (ie the -// public key) and the rpc port in any case. -func InitAlloc(metaDirPath string, initRPCPort int) (string, int, error) { - - initDirFor := func(path string) error { - dir := filepath.Dir(path) - return os.MkdirAll(dir, 0750) - } - - var err error - - exists := func(path string) bool { - - if err != nil { - return false - - } else if _, err = os.Stat(path); errors.Is(err, fs.ErrNotExist) { - err = nil - return false - - } else if err != nil { - err = fmt.Errorf("checking if %q exists: %w", path, err) - return false - } - - return true - } - - nodeKeyPath := nodeKeyPath(metaDirPath) - nodeKeyPubPath := nodeKeyPubPath(metaDirPath) - nodeRPCPortPath := nodeRPCPortPath(metaDirPath) - - nodeKeyPathExists := exists(nodeKeyPath) - nodeKeyPubPathExists := exists(nodeKeyPubPath) - nodeRPCPortPathExists := exists(nodeRPCPortPath) - - if err != nil { - return "", 0, err - - } else if nodeKeyPubPathExists != nodeKeyPathExists { - return "", 0, fmt.Errorf("%q or %q exist without the other existing", nodeKeyPath, nodeKeyPubPath) - - } - - var ( - pubKeyStr string - rpcPort int - ) - - if nodeKeyPathExists { - - if pubKeyStr, err = loadAllocID(metaDirPath); err != nil { - return "", 0, fmt.Errorf("reading node public key file: %w", err) - } - - } else { - - if err := initDirFor(nodeKeyPath); err != nil { - return "", 0, fmt.Errorf("creating directory for %q: %w", nodeKeyPath, err) - } - - pubKey, privKey := GeneratePeerKey() - - if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil { - return "", 0, fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err) - - } else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil { - return "", 0, fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err) - } - - pubKeyStr = hex.EncodeToString(pubKey) - } - - if nodeRPCPortPathExists { - - if rpcPortStr, err := os.ReadFile(nodeRPCPortPath); err != nil { - return "", 0, fmt.Errorf("reading rpc port from %q: %w", nodeRPCPortPath, err) - - } else if rpcPort, err = strconv.Atoi(string(rpcPortStr)); err != nil { - return "", 0, fmt.Errorf("parsing rpc port %q from %q: %w", rpcPortStr, nodeRPCPortPath, err) - } - - } else { - - if err := initDirFor(nodeRPCPortPath); err != nil { - return "", 0, fmt.Errorf("creating directory for %q: %w", nodeRPCPortPath, err) - } - - rpcPortStr := strconv.Itoa(initRPCPort) - - if err := os.WriteFile(nodeRPCPortPath, []byte(rpcPortStr), 0440); err != nil { - return "", 0, fmt.Errorf("writing rpc port %q to %q: %w", rpcPortStr, nodeRPCPortPath, err) - } - - rpcPort = initRPCPort - } - - return pubKeyStr, rpcPort, nil -} diff --git a/go/garage/garagesrv/garagesrv.go b/go/garage/garagesrv/garagesrv.go new file mode 100644 index 0000000..c73a93b --- /dev/null +++ b/go/garage/garagesrv/garagesrv.go @@ -0,0 +1,152 @@ +// Package garage contains helper functions and types which are useful for +// setting up garage configs, processes, and deployments. +package garagesrv + +import ( + "crypto/ed25519" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strconv" +) + +func nodeKeyPath(metaDirPath string) string { + return filepath.Join(metaDirPath, "node_key") +} + +func nodeKeyPubPath(metaDirPath string) string { + return filepath.Join(metaDirPath, "node_key.pub") +} + +func nodeRPCPortPath(metaDirPath string) string { + return filepath.Join(metaDirPath, "isle", "rpc_port") +} + +// loadAllocID returns the peer ID (ie the public key) of the node at the given +// meta directory. +func loadAllocID(metaDirPath string) (string, error) { + nodeKeyPubPath := nodeKeyPubPath(metaDirPath) + + pubKey, err := os.ReadFile(nodeKeyPubPath) + if err != nil { + return "", fmt.Errorf("reading %q: %w", nodeKeyPubPath, err) + } + + return hex.EncodeToString(pubKey), nil +} + +// generatePeerKey generates and returns a public/private key pair for a garage +// instance. +func generatePeerKey() (pubKey, privKey []byte) { + pubKey, privKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + panic(err) + } + + return pubKey, privKey +} + +// InitAlloc initializes the meta directory and keys for a particular +// allocation, if it hasn't been done so already. It returns the peer ID (ie the +// public key) and the rpc port in any case. +func InitAlloc(metaDirPath string, initRPCPort int) (string, int, error) { + + initDirFor := func(path string) error { + dir := filepath.Dir(path) + return os.MkdirAll(dir, 0750) + } + + var err error + + exists := func(path string) bool { + + if err != nil { + return false + + } else if _, err = os.Stat(path); errors.Is(err, fs.ErrNotExist) { + err = nil + return false + + } else if err != nil { + err = fmt.Errorf("checking if %q exists: %w", path, err) + return false + } + + return true + } + + nodeKeyPath := nodeKeyPath(metaDirPath) + nodeKeyPubPath := nodeKeyPubPath(metaDirPath) + nodeRPCPortPath := nodeRPCPortPath(metaDirPath) + + nodeKeyPathExists := exists(nodeKeyPath) + nodeKeyPubPathExists := exists(nodeKeyPubPath) + nodeRPCPortPathExists := exists(nodeRPCPortPath) + + if err != nil { + return "", 0, err + + } else if nodeKeyPubPathExists != nodeKeyPathExists { + return "", 0, fmt.Errorf("%q or %q exist without the other existing", nodeKeyPath, nodeKeyPubPath) + + } + + var ( + pubKeyStr string + rpcPort int + ) + + if nodeKeyPathExists { + + if pubKeyStr, err = loadAllocID(metaDirPath); err != nil { + return "", 0, fmt.Errorf("reading node public key file: %w", err) + } + + } else { + + if err := initDirFor(nodeKeyPath); err != nil { + return "", 0, fmt.Errorf("creating directory for %q: %w", nodeKeyPath, err) + } + + pubKey, privKey := generatePeerKey() + + if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil { + return "", 0, fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err) + + } else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil { + return "", 0, fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err) + } + + pubKeyStr = hex.EncodeToString(pubKey) + } + + if nodeRPCPortPathExists { + + if rpcPortStr, err := os.ReadFile(nodeRPCPortPath); err != nil { + return "", 0, fmt.Errorf("reading rpc port from %q: %w", nodeRPCPortPath, err) + + } else if rpcPort, err = strconv.Atoi(string(rpcPortStr)); err != nil { + return "", 0, fmt.Errorf("parsing rpc port %q from %q: %w", rpcPortStr, nodeRPCPortPath, err) + } + + } else { + + if err := initDirFor(nodeRPCPortPath); err != nil { + return "", 0, fmt.Errorf("creating directory for %q: %w", nodeRPCPortPath, err) + } + + rpcPortStr := strconv.Itoa(initRPCPort) + + if err := os.WriteFile(nodeRPCPortPath, []byte(rpcPortStr), 0440); err != nil { + return "", 0, fmt.Errorf("writing rpc port %q to %q: %w", rpcPortStr, nodeRPCPortPath, err) + } + + rpcPort = initRPCPort + } + + return pubKeyStr, rpcPort, nil +} diff --git a/go/garage/tpl.go b/go/garage/garagesrv/tpl.go similarity index 87% rename from go/garage/tpl.go rename to go/garage/garagesrv/tpl.go index e0dedc8..adb7df3 100644 --- a/go/garage/tpl.go +++ b/go/garage/garagesrv/tpl.go @@ -1,10 +1,13 @@ -package garage +package garagesrv import ( "fmt" "io" "os" + "strconv" "text/template" + + "isle/garage" ) // GarageTomlData describes all fields needed for rendering a garage.toml @@ -16,11 +19,8 @@ type GarageTomlData struct { RPCSecret string AdminToken string - RPCAddr string - S3APIAddr string - AdminAddr string - - BootstrapPeers []string + garage.LocalPeer + BootstrapPeers []garage.RemotePeer } var garageTomlTpl = template.Must(template.New("").Parse(` @@ -28,14 +28,14 @@ var garageTomlTpl = template.Must(template.New("").Parse(` metadata_dir = "{{ .MetaPath }}" data_dir = "{{ .DataPath }}" -replication_mode = "3" +replication_mode = "` + strconv.Itoa(garage.ReplicationFactor) + `" rpc_secret = "{{ .RPCSecret }}" rpc_bind_addr = "{{ .RPCAddr }}" rpc_public_addr = "{{ .RPCAddr }}" bootstrap_peers = [{{- range .BootstrapPeers }} -"{{ . }}", +"{{ .RPCPeerAddr }}", {{ end -}}] [s3_api] diff --git a/go/garage/peer.go b/go/garage/peer.go index f2fa8cd..c58479b 100644 --- a/go/garage/peer.go +++ b/go/garage/peer.go @@ -1,8 +1,6 @@ package garage import ( - "crypto/ed25519" - "crypto/rand" "fmt" "net" "strconv" @@ -24,17 +22,6 @@ type LocalPeer struct { AdminPort int } -// GeneratePeerKey generates and returns a public/private key pair for a garage -// instance. -func GeneratePeerKey() (pubKey, privKey []byte) { - pubKey, privKey, err := ed25519.GenerateKey(rand.Reader) - if err != nil { - panic(err) - } - - return pubKey, privKey -} - // RPCAddr returns the address of the peer's RPC port. func (p RemotePeer) RPCAddr() string { return net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort))