micropelago/isle
4
1
Fork 0
Code Issues 1 Pull Requests Projects Releases 5 Activity

Use a real private key for garage instances

Browse Source
This commit is contained in:
Brian Picciano 2022-10-29 00:09:18 +02:00
parent b26f4bdd6a
commit 711d568036
9 changed files with 163 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
entrypoint/src/bootstrap/garage.go
View File

@ -12,9 +12,9 @@ const (
)
// GaragePeers returns a Peer for each known garage instance in the network.
func (b Bootstrap) GaragePeers() []garage.Peer {
func (b Bootstrap) GaragePeers() []garage.RemotePeer {
var peers []garage.Peer
var peers []garage.RemotePeer
for _, host := range b.Hosts {
@ -24,7 +24,8 @@ func (b Bootstrap) GaragePeers() []garage.Peer {
for _, instance := range host.Garage.Instances {
peer := garage.Peer{
peer := garage.RemotePeer{
ID: instance.ID,
IP: host.Nebula.IP,
RPCPort: instance.RPCPort,
S3APIPort: instance.S3APIPort,
@ -50,13 +51,14 @@ func (b Bootstrap) GarageRPCPeerAddrs() []string {
// ChooseGaragePeer returns a Peer for a garage instance from the network. It
// will prefer a garage instance on this particular host, if there is one, but
// will otherwise return a random endpoint.
func (b Bootstrap) ChooseGaragePeer() garage.Peer {
func (b Bootstrap) ChooseGaragePeer() garage.RemotePeer {
thisHost := b.ThisHost()
if thisHost.Garage != nil && len(thisHost.Garage.Instances) > 0 {
inst := thisHost.Garage.Instances[0]
return garage.Peer{
return garage.RemotePeer{
ID: inst.ID,
IP: thisHost.Nebula.IP,
RPCPort: inst.RPCPort,
S3APIPort: inst.S3APIPort,

5
entrypoint/src/bootstrap/hosts.go
View File

@ -22,8 +22,9 @@ type NebulaHost struct {
// GarageHost describes a single garage instance in the GarageHost.
type GarageHostInstance struct {
RPCPort int `yaml:"rpc_port"`
S3APIPort int `yaml:"s3_api_port"`
ID string `yaml:"id"`
RPCPort int `yaml:"rpc_port"`
S3APIPort int `yaml:"s3_api_port"`
}
// GarageHost describes the garage configuration of a Host which is relevant for

8
entrypoint/src/cmd/entrypoint/daemon_util.go
View File

@ -4,6 +4,7 @@ import (
"context"
"cryptic-net/bootstrap"
"cryptic-net/daemon"
"cryptic-net/garage"
"fmt"
"time"
)
@ -25,7 +26,14 @@ func mergeDaemonConfigIntoBootstrap(
host.Garage = new(bootstrap.GarageHost)
for _, alloc := range allocs {
id, err := garage.InitAlloc(alloc.MetaPath)
if err != nil {
return bootstrap.Bootstrap{}, fmt.Errorf("initializing alloc at %q: %w", alloc.MetaPath, err)
}
host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
ID: id,
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
})

64
entrypoint/src/cmd/entrypoint/garage_util.go
View File

@ -7,7 +7,6 @@ import (
"cryptic-net/garage"
"fmt"
"net"
"os"
"path/filepath"
"strconv"
@ -66,6 +65,25 @@ func waitForGarageAndNebula(
}
// bootstrapGarageHostForAlloc returns the bootstrap.GarageHostInstance which
// corresponds with the given alloc from the daemon config. This will panic if
// no associated instance can be found.
//
// This assumes that mergeDaemonConfigIntoBootstrap has already been called.
func bootstrapGarageHostForAlloc(
host bootstrap.Host,
alloc daemon.ConfigStorageAllocation,
) bootstrap.GarageHostInstance {
for _, inst := range host.Garage.Instances {
if inst.RPCPort == alloc.RPCPort {
return inst
}
}
panic(fmt.Sprintf("could not find alloc %+v in the bootstrap data", alloc))
}
func garageWriteChildConfig(
hostBootstrap bootstrap.Bootstrap,
alloc daemon.ConfigStorageAllocation,
@ -73,28 +91,17 @@ func garageWriteChildConfig(
string, error,
) {
if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
}
thisHost := hostBootstrap.ThisHost()
id := bootstrapGarageHostForAlloc(thisHost, alloc).ID
peer := garage.Peer{
IP: thisHost.Nebula.IP,
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
}
pubKey, privKey := peer.RPCPeerKey()
nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
peer := garage.LocalPeer{
RemotePeer: garage.RemotePeer{
ID: id,
IP: thisHost.Nebula.IP,
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
},
AdminPort: alloc.AdminPort,
}
garageTomlPath := filepath.Join(
@ -108,9 +115,9 @@ func garageWriteChildConfig(
RPCSecret: hostBootstrap.GarageRPCSecret,
AdminToken: hostBootstrap.GarageAdminToken,
RPCAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
APIAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
AdminAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.AdminPort)),
RPCAddr: peer.RPCAddr(),
S3APIAddr: peer.S3APIAddr(),
AdminAddr: peer.AdminAddr(),
BootstrapPeers: hostBootstrap.GarageRPCPeerAddrs(),
})
@ -224,7 +231,6 @@ func garageApplyLayout(
adminClient = newGarageAdminClient(hostBootstrap, daemonConfig)
thisHost = hostBootstrap.ThisHost()
hostName = thisHost.Name
ip = thisHost.Nebula.IP
allocs = daemonConfig.Storage.Allocations
)
@ -239,13 +245,9 @@ func garageApplyLayout(
for _, alloc := range allocs {
peer := garage.Peer{
IP: ip,
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
}
id := bootstrapGarageHostForAlloc(thisHost, alloc).ID
clusterLayout[peer.RPCPeerID()] = peerLayout{
clusterLayout[id] = peerLayout{
Capacity: alloc.Capacity / 100,
Zone: hostName,
Tags: []string{},

87
entrypoint/src/garage/garage.go
View File

@ -2,6 +2,15 @@
// setting up garage configs, processes, and deployments.
package garage
import (
"encoding/hex"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
)
const (
// Region is the region which garage is configured with.
@ -15,3 +24,81 @@ const (
// cluster. We currently only support a factor of 3.
ReplicationFactor = 3
)
func nodeKeyPath(metaDirPath string) string {
return filepath.Join(metaDirPath, "node_key")
}
func nodeKeyPubPath(metaDirPath string) string {
return filepath.Join(metaDirPath, "node_key.pub")
}
// LoadAllocID returns the peer ID (ie the public key) of the node at the given
// meta directory.
func LoadAllocID(metaDirPath string) (string, error) {
nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
pubKey, err := os.ReadFile(nodeKeyPubPath)
if err != nil {
return "", fmt.Errorf("reading %q: %w", nodeKeyPubPath, err)
}
return hex.EncodeToString(pubKey), nil
}
// InitAlloc initializes the meta directory and keys for a particular
// allocation, if it hasn't been done so already. It returns the peer ID (ie the
// public key) in any case.
func InitAlloc(metaDirPath string) (string, error) {
var err error
exists := func(path string) bool {
if err != nil {
return false
} else if _, err = os.Stat(path); errors.Is(err, fs.ErrNotExist) {
return false
} else if err != nil {
err = fmt.Errorf("checking if %q exists: %w", path, err)
return false
}
return true
}
nodeKeyPath := nodeKeyPath(metaDirPath)
nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
nodeKeyPathExists := exists(nodeKeyPath)
nodeKeyPubPathExists := exists(nodeKeyPubPath)
if err != nil {
return "", err
} else if nodeKeyPubPathExists != nodeKeyPathExists {
return "", fmt.Errorf("%q or %q exist without the other existing", nodeKeyPath, nodeKeyPubPath)
} else if nodeKeyPathExists {
return LoadAllocID(metaDirPath)
}
// node key hasn't been written, write it
if err := os.MkdirAll(metaDirPath, 0750); err != nil {
return "", fmt.Errorf("making directory %q: %w", metaDirPath, err)
}
pubKey, privKey := GeneratePeerKey()
if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
}
return "", nil
}

41
entrypoint/src/garage/infinite_reader.go
View File

@ -1,41 +0,0 @@
package garage
import "io"
type infiniteReader struct {
b []byte
i int
}
// NewInfiniteReader returns a reader which will produce the given bytes in
// repetition. len(b) must be greater than 0.
func NewInfiniteReader(b []byte) io.Reader {
if len(b) == 0 {
panic("len(b) must be greater than 0")
}
return &infiniteReader{b: b}
}
func (r *infiniteReader) Read(b []byte) (int, error) {
// here, have a puzzle
var n int
for {
n += copy(b[n:], r.b[r.i:])
if r.i > 0 {
n += copy(b[n:], r.b[:r.i])
}
r.i = (r.i + n) % len(r.b)
if n >= len(b) {
return n, nil
}
}
}

101
entrypoint/src/garage/infinite_reader_test.go
View File

@ -1,101 +0,0 @@
package garage
import (
"bytes"
"strconv"
"testing"
)
func TestInfiniteReader(t *testing.T) {
tests := []struct {
in []byte
size int
exp []string
}{
{
in: []byte("a"),
size: 1,
exp: []string{"a"},
},
{
in: []byte("ab"),
size: 1,
exp: []string{"a", "b"},
},
{
in: []byte("ab"),
size: 2,
exp: []string{"ab"},
},
{
in: []byte("ab"),
size: 3,
exp: []string{"aba", "bab"},
},
{
in: []byte("ab"),
size: 4,
exp: []string{"abab"},
},
{
in: []byte("ab"),
size: 5,
exp: []string{"ababa", "babab"},
},
{
in: []byte("abc"),
size: 1,
exp: []string{"a", "b", "c"},
},
{
in: []byte("abc"),
size: 2,
exp: []string{"ab", "ca", "bc"},
},
{
in: []byte("abc"),
size: 3,
exp: []string{"abc"},
},
{
in: []byte("abc"),
size: 4,
exp: []string{"abca", "bcab", "cabc"},
},
{
in: []byte("abc"),
size: 5,
exp: []string{"abcab", "cabca", "bcabc"},
},
}
for i, test := range tests {
t.Run(strconv.Itoa(i), func(t *testing.T) {
r := NewInfiniteReader(test.in)
buf := make([]byte, test.size)
assertRead := func(expBuf []byte) {
n, err := r.Read(buf)
if !bytes.Equal(buf, expBuf) {
t.Fatalf("expected bytes %q, got %q", expBuf, buf)
} else if n != len(buf) {
t.Fatalf("expected n %d, got %d", len(buf), n)
} else if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
for i := 0; i < 3; i++ {
for _, expStr := range test.exp {
assertRead([]byte(expStr))
}
}
})
}
}

55
entrypoint/src/garage/peer.go
View File

@ -2,35 +2,32 @@ package garage
import (
"crypto/ed25519"
"encoding/hex"
"crypto/rand"
"fmt"
"net"
"strconv"
)
// Peer describes all information necessary to connect to a given garage node.
type Peer struct {
// RemotePeer describes all information necessary to connect to a given garage
// node.
type RemotePeer struct {
ID string
IP string
RPCPort int
S3APIPort int
}
// RPCPeerKey deterministically generates a public/private keys which can
// be used as a garage node key.
//
// DANGER: This function will deterministically produce public/private keys
// given some arbitrary input. This is NEVER what you want. It's only being used
// in cryptic-net for a very specific purpose for which I think it's ok and is
// very necessary, and people are probably _still_ going to yell at me.
//
func (p Peer) RPCPeerKey() (pubKey, privKey []byte) {
input := []byte(net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort)))
// LocalPeer describes the configuration of a local garage instance.
type LocalPeer struct {
RemotePeer
// Append the length of the input to the input, so that the input "foo"
// doesn't generate the same key as the input "foofoo".
input = strconv.AppendInt(input, int64(len(input)), 10)
AdminPort int
}
pubKey, privKey, err := ed25519.GenerateKey(NewInfiniteReader(input))
// GeneratePeerKey generates and returns a public/private key pair for a garage
// instance.
func GeneratePeerKey() (pubKey, privKey []byte) {
pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
panic(err)
}
@ -38,29 +35,23 @@ func (p Peer) RPCPeerKey() (pubKey, privKey []byte) {
return pubKey, privKey
}
// RPCPeerID returns the peer ID of the garage node for use in communicating
// over RPC.
//
// DANGER: See warning on RPCPeerKey.
func (p Peer) RPCPeerID() string {
pubKey, _ := p.RPCPeerKey()
return hex.EncodeToString(pubKey)
}
// RPCAddr returns the address of the peer's RPC port.
func (p Peer) RPCAddr() string {
func (p RemotePeer) RPCAddr() string {
return net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort))
}
// RPCPeerAddr returns the full peer address (e.g. "id@ip:port") of the garage
// node for use in communicating over RPC.
//
// DANGER: See warning on RPCPeerKey.
func (p Peer) RPCPeerAddr() string {
return fmt.Sprintf("%s@%s", p.RPCPeerID(), p.RPCAddr())
func (p RemotePeer) RPCPeerAddr() string {
return fmt.Sprintf("%s@%s", p.ID, p.RPCAddr())
}
// S3APIAddr returns the address of the peer's S3 API port.
func (p Peer) S3APIAddr() string {
func (p RemotePeer) S3APIAddr() string {
return net.JoinHostPort(p.IP, strconv.Itoa(p.S3APIPort))
}
// AdminAddr returns the address of the peer's S3 API port.
func (p LocalPeer) AdminAddr() string {
return net.JoinHostPort(p.IP, strconv.Itoa(p.AdminPort))
}

4
entrypoint/src/garage/tpl.go
View File

@ -17,7 +17,7 @@ type GarageTomlData struct {
AdminToken string
RPCAddr string
APIAddr string
S3APIAddr string
AdminAddr string
BootstrapPeers []string
@ -39,7 +39,7 @@ bootstrap_peers = [{{- range .BootstrapPeers }}
{{ end -}}]
[s3_api]
api_bind_addr = "{{ .APIAddr }}"
api_bind_addr = "{{ .S3APIAddr }}"
s3_region = "garage"
[admin]