micropelago/isle
4
1
Fork 0
Code Issues 1 Pull Requests Projects Releases 5 Activity

Separate garage server logic into its own package

Browse Source
This commit is contained in:
Brian Picciano 2024-06-12 10:18:33 +02:00
parent dee4af012e
commit 842c169169
8 changed files with 168 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
go/bootstrap/garage.go
View File

@ -27,17 +27,6 @@ func (b Bootstrap) GaragePeers() []garage.RemotePeer {
return peers
}
// GarageRPCPeerAddrs returns the full RPC peer address for each known garage
// instance in the network.
func (b Bootstrap) GarageRPCPeerAddrs() []string {
var addrs []string
for _, peer := range b.GaragePeers() {
addr := peer.RPCPeerAddr()
addrs = append(addrs, addr)
}
return addrs
}
// ChooseGaragePeer returns a Peer for a garage instance from the network. It
// will prefer a garage instance on this particular host, if there is one, but
// will otherwise return a random endpoint.

4
go/cmd/entrypoint/daemon_util.go
View File

@ -5,7 +5,7 @@ import (
"fmt"
"isle/bootstrap"
"isle/daemon"
"isle/garage"
"isle/garage/garagesrv"
"time"
)
@ -29,7 +29,7 @@ func coalesceDaemonConfigAndBootstrap(
for i, alloc := range allocs {
id, rpcPort, err := garage.InitAlloc(alloc.MetaPath, alloc.RPCPort)
id, rpcPort, err := garagesrv.InitAlloc(alloc.MetaPath, alloc.RPCPort)
if err != nil {
return bootstrap.Bootstrap{}, daemon.Config{}, fmt.Errorf("initializing alloc at %q: %w", alloc.MetaPath, err)
}

10
go/cmd/entrypoint/garage_util.go
View File

@ -6,6 +6,7 @@ import (
"isle/bootstrap"
"isle/daemon"
"isle/garage"
"isle/garage/garagesrv"
"net"
"path/filepath"
"strconv"
@ -128,18 +129,15 @@ func garageWriteChildConfig(
envRuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
)
err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
err := garagesrv.WriteGarageTomlFile(garageTomlPath, garagesrv.GarageTomlData{
MetaPath: alloc.MetaPath,
DataPath: alloc.DataPath,
RPCSecret: hostBootstrap.Garage.RPCSecret,
AdminToken: hostBootstrap.Garage.AdminToken,
RPCAddr: peer.RPCAddr(),
S3APIAddr: peer.S3APIAddr(),
AdminAddr: peer.AdminAddr(),
BootstrapPeers: hostBootstrap.GarageRPCPeerAddrs(),
LocalPeer: peer,
BootstrapPeers: hostBootstrap.GaragePeers(),
})
if err != nil {

10
go/garage/client.go
View File

@ -1,8 +1,6 @@
package garage
import (
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
@ -10,14 +8,6 @@ import (
"github.com/minio/minio-go/v7/pkg/credentials"
)
func randStr(l int) string {
b := make([]byte, l)
if _, err := rand.Read(b); err != nil {
panic(err)
}
return hex.EncodeToString(b)
}
// IsKeyNotFound returns true if the given error is the result of a key not
// being found in a bucket.
func IsKeyNotFound(err error) bool {

140
go/garage/garage.go
View File

@ -1,17 +1,7 @@
// Package garage contains helper functions and types which are useful for
// setting up garage configs, processes, and deployments.
// Package garage contains types and helpers related to interacting with garage
// processes via garage's APIs.
package garage
import (
"encoding/hex"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strconv"
)
const (
// Region is the region which garage is configured with.
@ -25,129 +15,3 @@ const (
// cluster. We currently only support a factor of 3.
ReplicationFactor = 3
)
func nodeKeyPath(metaDirPath string) string {
return filepath.Join(metaDirPath, "node_key")
}
func nodeKeyPubPath(metaDirPath string) string {
return filepath.Join(metaDirPath, "node_key.pub")
}
func nodeRPCPortPath(metaDirPath string) string {
return filepath.Join(metaDirPath, "isle", "rpc_port")
}
// loadAllocID returns the peer ID (ie the public key) of the node at the given
// meta directory.
func loadAllocID(metaDirPath string) (string, error) {
nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
pubKey, err := os.ReadFile(nodeKeyPubPath)
if err != nil {
return "", fmt.Errorf("reading %q: %w", nodeKeyPubPath, err)
}
return hex.EncodeToString(pubKey), nil
}
// InitAlloc initializes the meta directory and keys for a particular
// allocation, if it hasn't been done so already. It returns the peer ID (ie the
// public key) and the rpc port in any case.
func InitAlloc(metaDirPath string, initRPCPort int) (string, int, error) {
initDirFor := func(path string) error {
dir := filepath.Dir(path)
return os.MkdirAll(dir, 0750)
}
var err error
exists := func(path string) bool {
if err != nil {
return false
} else if _, err = os.Stat(path); errors.Is(err, fs.ErrNotExist) {
err = nil
return false
} else if err != nil {
err = fmt.Errorf("checking if %q exists: %w", path, err)
return false
}
return true
}
nodeKeyPath := nodeKeyPath(metaDirPath)
nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
nodeRPCPortPath := nodeRPCPortPath(metaDirPath)
nodeKeyPathExists := exists(nodeKeyPath)
nodeKeyPubPathExists := exists(nodeKeyPubPath)
nodeRPCPortPathExists := exists(nodeRPCPortPath)
if err != nil {
return "", 0, err
} else if nodeKeyPubPathExists != nodeKeyPathExists {
return "", 0, fmt.Errorf("%q or %q exist without the other existing", nodeKeyPath, nodeKeyPubPath)
}
var (
pubKeyStr string
rpcPort int
)
if nodeKeyPathExists {
if pubKeyStr, err = loadAllocID(metaDirPath); err != nil {
return "", 0, fmt.Errorf("reading node public key file: %w", err)
}
} else {
if err := initDirFor(nodeKeyPath); err != nil {
return "", 0, fmt.Errorf("creating directory for %q: %w", nodeKeyPath, err)
}
pubKey, privKey := GeneratePeerKey()
if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
return "", 0, fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
return "", 0, fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
}
pubKeyStr = hex.EncodeToString(pubKey)
}
if nodeRPCPortPathExists {
if rpcPortStr, err := os.ReadFile(nodeRPCPortPath); err != nil {
return "", 0, fmt.Errorf("reading rpc port from %q: %w", nodeRPCPortPath, err)
} else if rpcPort, err = strconv.Atoi(string(rpcPortStr)); err != nil {
return "", 0, fmt.Errorf("parsing rpc port %q from %q: %w", rpcPortStr, nodeRPCPortPath, err)
}
} else {
if err := initDirFor(nodeRPCPortPath); err != nil {
return "", 0, fmt.Errorf("creating directory for %q: %w", nodeRPCPortPath, err)
}
rpcPortStr := strconv.Itoa(initRPCPort)
if err := os.WriteFile(nodeRPCPortPath, []byte(rpcPortStr), 0440); err != nil {
return "", 0, fmt.Errorf("writing rpc port %q to %q: %w", rpcPortStr, nodeRPCPortPath, err)
}
rpcPort = initRPCPort
}
return pubKeyStr, rpcPort, nil
}

152
go/garage/garagesrv/garagesrv.go Normal file
View File

@ -0,0 +1,152 @@
// Package garage contains helper functions and types which are useful for
// setting up garage configs, processes, and deployments.
package garagesrv
import (
"crypto/ed25519"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strconv"
)
func nodeKeyPath(metaDirPath string) string {
return filepath.Join(metaDirPath, "node_key")
}
func nodeKeyPubPath(metaDirPath string) string {
return filepath.Join(metaDirPath, "node_key.pub")
}
func nodeRPCPortPath(metaDirPath string) string {
return filepath.Join(metaDirPath, "isle", "rpc_port")
}
// loadAllocID returns the peer ID (ie the public key) of the node at the given
// meta directory.
func loadAllocID(metaDirPath string) (string, error) {
nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
pubKey, err := os.ReadFile(nodeKeyPubPath)
if err != nil {
return "", fmt.Errorf("reading %q: %w", nodeKeyPubPath, err)
}
return hex.EncodeToString(pubKey), nil
}
// generatePeerKey generates and returns a public/private key pair for a garage
// instance.
func generatePeerKey() (pubKey, privKey []byte) {
pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
panic(err)
}
return pubKey, privKey
}
// InitAlloc initializes the meta directory and keys for a particular
// allocation, if it hasn't been done so already. It returns the peer ID (ie the
// public key) and the rpc port in any case.
func InitAlloc(metaDirPath string, initRPCPort int) (string, int, error) {
initDirFor := func(path string) error {
dir := filepath.Dir(path)
return os.MkdirAll(dir, 0750)
}
var err error
exists := func(path string) bool {
if err != nil {
return false
} else if _, err = os.Stat(path); errors.Is(err, fs.ErrNotExist) {
err = nil
return false
} else if err != nil {
err = fmt.Errorf("checking if %q exists: %w", path, err)
return false
}
return true
}
nodeKeyPath := nodeKeyPath(metaDirPath)
nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
nodeRPCPortPath := nodeRPCPortPath(metaDirPath)
nodeKeyPathExists := exists(nodeKeyPath)
nodeKeyPubPathExists := exists(nodeKeyPubPath)
nodeRPCPortPathExists := exists(nodeRPCPortPath)
if err != nil {
return "", 0, err
} else if nodeKeyPubPathExists != nodeKeyPathExists {
return "", 0, fmt.Errorf("%q or %q exist without the other existing", nodeKeyPath, nodeKeyPubPath)
}
var (
pubKeyStr string
rpcPort int
)
if nodeKeyPathExists {
if pubKeyStr, err = loadAllocID(metaDirPath); err != nil {
return "", 0, fmt.Errorf("reading node public key file: %w", err)
}
} else {
if err := initDirFor(nodeKeyPath); err != nil {
return "", 0, fmt.Errorf("creating directory for %q: %w", nodeKeyPath, err)
}
pubKey, privKey := generatePeerKey()
if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
return "", 0, fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
return "", 0, fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
}
pubKeyStr = hex.EncodeToString(pubKey)
}
if nodeRPCPortPathExists {
if rpcPortStr, err := os.ReadFile(nodeRPCPortPath); err != nil {
return "", 0, fmt.Errorf("reading rpc port from %q: %w", nodeRPCPortPath, err)
} else if rpcPort, err = strconv.Atoi(string(rpcPortStr)); err != nil {
return "", 0, fmt.Errorf("parsing rpc port %q from %q: %w", rpcPortStr, nodeRPCPortPath, err)
}
} else {
if err := initDirFor(nodeRPCPortPath); err != nil {
return "", 0, fmt.Errorf("creating directory for %q: %w", nodeRPCPortPath, err)
}
rpcPortStr := strconv.Itoa(initRPCPort)
if err := os.WriteFile(nodeRPCPortPath, []byte(rpcPortStr), 0440); err != nil {
return "", 0, fmt.Errorf("writing rpc port %q to %q: %w", rpcPortStr, nodeRPCPortPath, err)
}
rpcPort = initRPCPort
}
return pubKeyStr, rpcPort, nil
}

16
go/garage/tpl.go → go/garage/garagesrv/tpl.go
View File

@ -1,10 +1,13 @@
package garage
package garagesrv
import (
"fmt"
"io"
"os"
"strconv"
"text/template"
"isle/garage"
)
// GarageTomlData describes all fields needed for rendering a garage.toml
@ -16,11 +19,8 @@ type GarageTomlData struct {
RPCSecret string
AdminToken string
RPCAddr string
S3APIAddr string
AdminAddr string
BootstrapPeers []string
garage.LocalPeer
BootstrapPeers []garage.RemotePeer
}
var garageTomlTpl = template.Must(template.New("").Parse(`
@ -28,14 +28,14 @@ var garageTomlTpl = template.Must(template.New("").Parse(`
metadata_dir = "{{ .MetaPath }}"
data_dir = "{{ .DataPath }}"
replication_mode = "3"
replication_mode = "` + strconv.Itoa(garage.ReplicationFactor) + `"
rpc_secret = "{{ .RPCSecret }}"
rpc_bind_addr = "{{ .RPCAddr }}"
rpc_public_addr = "{{ .RPCAddr }}"
bootstrap_peers = [{{- range .BootstrapPeers }}
"{{ . }}",
"{{ .RPCPeerAddr }}",
{{ end -}}]
[s3_api]

13
go/garage/peer.go
View File

@ -1,8 +1,6 @@
package garage
import (
"crypto/ed25519"
"crypto/rand"
"fmt"
"net"
"strconv"
@ -24,17 +22,6 @@ type LocalPeer struct {
AdminPort int
}
// GeneratePeerKey generates and returns a public/private key pair for a garage
// instance.
func GeneratePeerKey() (pubKey, privKey []byte) {
pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
panic(err)
}
return pubKey, privKey
}
// RPCAddr returns the address of the peer's RPC port.
func (p RemotePeer) RPCAddr() string {
return net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort))