micropelago/isle
4
1
Fork 0
Code Issues 1 Pull Requests Projects Releases 6 Activity

Some large inter-related refactors, moving towards network creation command

Browse Source 
Host types have been moved within the `bootstrap` package.

Refactored how boostrap FS is interacted with. There is now a
`Bootstrap` struct which has pre-loaded all data within the bootstrap
FS. This helps centralize the logic around reading the data (though not
yet completely).

Choosing of the garage node to interact with no longer occurs in like
three different places. It occurs at the environment level now, and is
aided by the new `garage.Peer` type.
This commit is contained in:
Brian Picciano 2022-10-15 16:28:03 +02:00
parent 004be0c2aa
commit 836e69735d
23 changed files with 479 additions and 479 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

191
go-workspace/src/bootstrap/bootstrap.go
View File

@ -2,130 +2,129 @@
package bootstrap
import (
"context"
crypticnet "cryptic-net"
"cryptic-net/garage"
"cryptic-net/nebula"
"cryptic-net/tarutil"
"cryptic-net/yamlutil"
"fmt"
"io"
"io/fs"
"os"
"strings"
)
// GetHashFromFS returns the hash of the contents of the given bootstrap file.
// It may return nil if the bootstrap file doesn't have a hash.
func GetHashFromFS(bootstrapFS fs.FS) ([]byte, error) {
// Paths within the bootstrap FS which for general data.
const (
HostNamePath = "hostname"
)
b, err := fs.ReadFile(bootstrapFS, tarutil.HashBinPath)
// Bootstrap is used for accessing all information contained within a
// bootstrap.tgz file.
//
// An instance of Bootstrap is read-only, the creator sub-package should be used
// to create new instances.
type Bootstrap struct {
Hosts map[string]Host
HostName string
if err != nil {
return nil, fmt.Errorf("reading file %q from bootstrap fs: %w", tarutil.HashBinPath, err)
NebulaCertsCACert string
NebulaCertsHostCert string
NebulaCertsHostKey string
GarageRPCSecret string
GarageGlobalBucketS3APICredentials garage.S3APICredentials
// Hash is a determinstic hash of the contents of the bootstrap file. This
// will be populated when parsing a Bootstrap from a bootstrap.tgz, but will
// be ignored when creating a new bootstrap.tgz.
Hash []byte
// DEPRECATED do not use
FS fs.FS
}
// FromFS loads a Boostrap instance from the given fs.FS, which presumably
// represents the file structure of a bootstrap.tgz file.
func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
var (
b Bootstrap
err error
)
b.FS = bootstrapFS
if b.Hosts, err = loadHosts(bootstrapFS); err != nil {
return Bootstrap{}, fmt.Errorf("loading hosts info from fs: %w", err)
}
if err = yamlutil.LoadYamlFSFile(
&b.GarageGlobalBucketS3APICredentials,
bootstrapFS,
GarageGlobalBucketKeyYmlPath,
); err != nil {
return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", b.GarageGlobalBucketS3APICredentials, err)
}
filesToLoadAsString := []struct {
into *string
path string
}{
{&b.HostName, HostNamePath},
{&b.NebulaCertsCACert, NebulaCertsCACertPath},
{&b.NebulaCertsHostCert, NebulaCertsHostCertPath},
{&b.NebulaCertsHostKey, NebulaCertsHostKeyPath},
{&b.GarageRPCSecret, GarageRPCSecretPath},
}
for _, f := range filesToLoadAsString {
body, err := fs.ReadFile(bootstrapFS, f.path)
if err != nil {
return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", f.path, err)
}
*f.into = string(body)
}
// TODO confirm if this is necessary
b.GarageRPCSecret = strings.TrimSpace(b.GarageRPCSecret)
if b.Hash, err = fs.ReadFile(bootstrapFS, tarutil.HashBinPath); err != nil {
return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", tarutil.HashBinPath, err)
}
return b, nil
}
// GetHashFromReader reads the given tgz file as an fs.FS, and passes that to
// GetHashFromFS.
func GetHashFromReader(r io.Reader) ([]byte, error) {
// FromReader reads a bootstrap.tgz file from the given io.Reader.
func FromReader(r io.Reader) (Bootstrap, error) {
bootstrapFS, err := tarutil.FSFromReader(r)
fs, err := tarutil.FSFromReader(r)
if err != nil {
return nil, fmt.Errorf("reading tar fs from reader: %w", err)
return Bootstrap{}, fmt.Errorf("reading bootstrap.tgz: %w", err)
}
return GetHashFromFS(bootstrapFS)
return FromFS(fs)
}
func newBootstrap(
ctx context.Context,
into io.Writer,
hostname provider,
nebulaCerts provider,
nebulaHosts provider,
garageRPCSecret provider,
garageGlobalBucketKey provider,
garageHosts provider,
) error {
// FromFile reads a bootstrap.tgz from a file at the given path.
func FromFile(path string) (Bootstrap, error) {
pairs := []struct {
path string
provider provider
}{
{"hostname", hostname},
{"nebula/certs", nebulaCerts},
{"nebula/hosts", nebulaHosts},
{"garage/rpc-secret.txt", garageRPCSecret},
{"garage/global-bucket-key.yml", garageGlobalBucketKey},
{"garage/hosts", garageHosts},
}
w := tarutil.NewTGZWriter(into)
for _, pair := range pairs {
if err := pair.provider(ctx, w, pair.path); err != nil {
return fmt.Errorf("populating %q in new bootstrap: %w", pair.path, err)
}
}
return w.Close()
}
// NewForThisHost generates a new bootstrap file for the current host, based on
// the existing environment as well as data in garage.
func NewForThisHost(env *crypticnet.Env, into io.Writer) error {
client, err := garage.GlobalBucketAPIClient(env)
f, err := os.Open(path)
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
return Bootstrap{}, fmt.Errorf("opening file: %w", err)
}
defer f.Close()
return newBootstrap(
env.Context,
into,
provideFromFS(env.BootstrapFS), // hostname
provideDirFromFS(env.BootstrapFS), // nebulaCerts
provideDirFromGarage(client), // nebulaHosts
provideFromFS(env.BootstrapFS), // garageRPCSecret
provideFromFS(env.BootstrapFS), // garageGlobalBucketKey
provideDirFromGarage(client), // garageHosts
)
return FromReader(f)
}
// NewForHost generates a new bootstrap file for an arbitrary host, based on the
// given admin file's FS and data in garage.
func NewForHost(env *crypticnet.Env, adminFS fs.FS, name string, into io.Writer) error {
// ThisHost is a shortcut for b.Hosts[b.HostName], but will panic if the
// HostName isn't found in the Hosts map.
func (b Bootstrap) ThisHost() Host {
host, ok := env.Hosts[name]
host, ok := b.Hosts[b.HostName]
if !ok {
return fmt.Errorf("unknown host %q, make sure host entry has been created", name)
panic(fmt.Sprintf("hostname %q not defined in bootstrap's hosts", b.HostName))
}
client, err := garage.GlobalBucketAPIClient(env)
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
nebulaHostCert, err := nebula.NewHostCert(adminFS, host.Nebula)
if err != nil {
return fmt.Errorf("creating new nebula host key/cert: %w", err)
}
nebulaHostCertDir := map[string][]byte{
"ca.crt": nebulaHostCert.CACert,
"host.key": nebulaHostCert.HostKey,
"host.crt": nebulaHostCert.HostCert,
}
return newBootstrap(
env.Context,
into,
provideFromBytes([]byte(name)), // hostname
provideDirFromMap(nebulaHostCertDir), // nebulaCerts
provideDirFromGarage(client), // nebulaHosts
provideFromFS(adminFS), // garageRPCSecret
provideFromFS(adminFS), // garageGlobalBucketKey
provideDirFromGarage(client), // garageHosts
)
return host
}

110
go-workspace/src/bootstrap/creator/creator.go Normal file
View File

@ -0,0 +1,110 @@
// Package creator is responsible for creating bootstrap files. It exists
// separately from the main bootstrap package in order to prevent import loops
// due to its use of crypticnet.Env.
package creator
import (
"context"
crypticnet "cryptic-net"
"cryptic-net/bootstrap"
"cryptic-net/nebula"
"cryptic-net/tarutil"
"fmt"
"io"
"io/fs"
)
func newBootstrap(
ctx context.Context,
into io.Writer,
hostname provider,
nebulaCertsCACert provider,
nebulaCertsHostCert provider,
nebulaCertsHostKey provider,
nebulaHosts provider,
garageRPCSecret provider,
garageGlobalBucketKey provider,
garageHosts provider,
) error {
pairs := []struct {
path string
provider provider
}{
{bootstrap.HostNamePath, hostname},
{bootstrap.NebulaCertsCACertPath, nebulaCertsCACert},
{bootstrap.NebulaCertsHostCertPath, nebulaCertsHostCert},
{bootstrap.NebulaCertsHostKeyPath, nebulaCertsHostKey},
{bootstrap.NebulaHostsDirPath, nebulaHosts},
{bootstrap.GarageRPCSecretPath, garageRPCSecret},
{bootstrap.GarageGlobalBucketKeyYmlPath, garageGlobalBucketKey},
{bootstrap.GarageHostsDirPath, garageHosts},
}
w := tarutil.NewTGZWriter(into)
for _, pair := range pairs {
if err := pair.provider(ctx, w, pair.path); err != nil {
return fmt.Errorf("populating %q in new bootstrap: %w", pair.path, err)
}
}
return w.Close()
}
// NewForThisHost generates a new bootstrap file for the current host, based on
// the existing environment as well as data in garage.
func NewForThisHost(env *crypticnet.Env, into io.Writer) error {
client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
return newBootstrap(
env.Context,
into,
provideFromFS(env.Bootstrap.FS), // hostname
provideFromFS(env.Bootstrap.FS), // nebulaCertsCACert
provideFromFS(env.Bootstrap.FS), // nebulaCertsHostCert
provideFromFS(env.Bootstrap.FS), // nebulaCertsHostKey
provideDirFromGarage(client), // nebulaHosts
provideFromFS(env.Bootstrap.FS), // garageRPCSecret
provideFromFS(env.Bootstrap.FS), // garageGlobalBucketKey
provideDirFromGarage(client), // garageHosts
)
}
// NewForHost generates a new bootstrap file for an arbitrary host, based on the
// given admin file's FS and data in garage.
func NewForHost(env *crypticnet.Env, adminFS fs.FS, name string, into io.Writer) error {
host, ok := env.Bootstrap.Hosts[name]
if !ok {
return fmt.Errorf("unknown host %q, make sure host entry has been created", name)
}
client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
nebulaHostCert, err := nebula.NewHostCert(adminFS, host.Nebula)
if err != nil {
return fmt.Errorf("creating new nebula host key/cert: %w", err)
}
return newBootstrap(
env.Context,
into,
provideFromBytes([]byte(name)), // hostname
provideFromBytes(nebulaHostCert.CACert), // nebulaCertsCACert
provideFromBytes(nebulaHostCert.HostCert), // nebulaCertsHostCert
provideFromBytes(nebulaHostCert.HostKey), // nebulaCertsHostKey
provideDirFromGarage(client), // nebulaHosts
provideFromFS(adminFS), // garageRPCSecret
provideFromFS(adminFS), // garageGlobalBucketKey
provideDirFromGarage(client), // garageHosts
)
}

20
go-workspace/src/bootstrap/provider.go → go-workspace/src/bootstrap/creator/provider.go
View File

@ -1,4 +1,4 @@
package bootstrap
package creator
import (
"context"
@ -6,7 +6,6 @@ import (
"cryptic-net/tarutil"
"fmt"
"io/fs"
"path/filepath"
"github.com/minio/minio-go/v7"
)
@ -69,23 +68,6 @@ func provideDirFromFS(srcFS fs.FS) provider {
}
}
func provideDirFromMap(m map[string][]byte) provider {
return func(
ctx context.Context,
w *tarutil.TGZWriter,
dirPath string,
) error {
for filePath, body := range m {
filePath := filepath.Join(dirPath, filePath)
w.WriteFileBytes(filePath, body)
}
return nil
}
}
// TODO it'd be great if we could wrap a minio.Client into an fs.FS. That would
// get rid of a weird dependency in this package, and clean up this code a ton.
func provideDirFromGarage(client *minio.Client) provider {

48
go-workspace/src/bootstrap/garage.go Normal file
View File

@ -0,0 +1,48 @@
package bootstrap
import (
"cryptic-net/garage"
)
// Paths within the bootstrap FS related to garage.
const (
GarageGlobalBucketKeyYmlPath = "garage/global-bucket-key.yml"
GarageRPCSecretPath = "garage/rpc-secret.txt"
GarageHostsDirPath = "garage/hosts"
)
// GaragePeers returns a Peer for each known garage instance in the network.
func (b Bootstrap) GaragePeers() []garage.Peer {
var peers []garage.Peer
for _, host := range b.Hosts {
if host.Garage == nil {
continue
}
for _, instance := range host.Garage.Instances {
peer := garage.Peer{
IP: host.Nebula.IP,
RPCPort: instance.RPCPort,
S3APIPort: instance.S3APIPort,
}
peers = append(peers, peer)
}
}
return peers
}
// GarageRPCPeerAddrs returns the full RPC peer address for each known garage
// instance in the network.
func (b Bootstrap) GarageRPCPeerAddrs() []string {
var addrs []string
for _, peer := range b.GaragePeers() {
addrs = append(addrs, peer.RPCPeerAddr())
}
return addrs
}

19
go-workspace/src/hosts.go → go-workspace/src/bootstrap/hosts.go
View File

@ -1,4 +1,4 @@
package crypticnet
package bootstrap
import (
"errors"
@ -19,9 +19,9 @@ type NebulaHost struct {
// GarageHostInstance describes a single garage instance running on a host.
type GarageHostInstance struct {
APIPort int `yaml:"api_port"`
RPCPort int `yaml:"rpc_port"`
WebPort int `yaml:"web_port"`
RPCPort int `yaml:"rpc_port"`
S3APIPort int `yaml:"s3_api_port"`
WebPort int `yaml:"web_port"`
}
// GarageHost describes the contents of a `./garage/hosts/<hostname>.yml` file.
@ -37,8 +37,7 @@ type Host struct {
Garage *GarageHost
}
// LostHosts returns a mapping of hostnames to Host objects for each host.
func LoadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
hosts := map[string]Host{}
@ -52,9 +51,11 @@ func LoadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
}
{
nebulaHostFiles, err := fs.Glob(bootstrapFS, "nebula/hosts/*.yml")
globPath := filepath.Join(NebulaHostsDirPath, "*.yml")
nebulaHostFiles, err := fs.Glob(bootstrapFS, globPath)
if err != nil {
return nil, fmt.Errorf("listing nebula host files: %w", err)
return nil, fmt.Errorf("listing nebula host files at %q in fs: %w", globPath, err)
}
for _, nebulaHostPath := range nebulaHostFiles {
@ -76,7 +77,7 @@ func LoadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
for hostName, host := range hosts {
garageHostPath := filepath.Join("garage/hosts", hostName+".yml")
garageHostPath := filepath.Join(GarageHostsDirPath, hostName+".yml")
var garageHost GarageHost
if err := readAsYaml(&garageHost, garageHostPath); errors.Is(err, fs.ErrNotExist) {

10
go-workspace/src/bootstrap/nebula.go Normal file
View File

@ -0,0 +1,10 @@
package bootstrap
// Paths within the bootstrap FS related to nebula.
const (
NebulaHostsDirPath = "nebula/hosts"
NebulaCertsCACertPath = "nebula/certs/ca.crt"
NebulaCertsHostCertPath = "nebula/certs/host.crt"
NebulaCertsHostKeyPath = "nebula/certs/host.key"
)

22
go-workspace/src/cmd/entrypoint/daemon.go
View File

@ -14,6 +14,7 @@ import (
crypticnet "cryptic-net"
"cryptic-net/bootstrap"
bootstrap_creator "cryptic-net/bootstrap/creator"
"cryptic-net/yamlutil"
"github.com/cryptic-io/pmux/pmuxlib"
@ -111,21 +112,16 @@ func reloadBootstrap(env *crypticnet.Env) (bool, error) {
buf := new(bytes.Buffer)
if err := bootstrap.NewForThisHost(env, buf); err != nil {
if err := bootstrap_creator.NewForThisHost(env, buf); err != nil {
return false, fmt.Errorf("generating new bootstrap from env: %w", err)
}
newHash, err := bootstrap.GetHashFromReader(bytes.NewReader(buf.Bytes()))
newBootstrap, err := bootstrap.FromReader(bytes.NewReader(buf.Bytes()))
if err != nil {
return false, fmt.Errorf("reading hash from new bootstrap file: %w", err)
return false, fmt.Errorf("parsing bootstrap which was just created: %w", err)
}
currHash, err := bootstrap.GetHashFromFS(env.BootstrapFS)
if err != nil {
return false, fmt.Errorf("reading hash from existing bootstrap fs: %w", err)
}
if bytes.Equal(newHash, currHash) {
if bytes.Equal(newBootstrap.Hash, env.Bootstrap.Hash) {
return false, nil
}
@ -136,12 +132,12 @@ func reloadBootstrap(env *crypticnet.Env) (bool, error) {
return true, nil
}
// runs a single pmux process for daemon, returning only once the env.Context
// runs a single pmux process ofor daemon, returning only once the env.Context
// has been canceled or bootstrap info has been changed. This will always block
// until the spawned pmux has returned.
func runDaemonPmuxOnce(env *crypticnet.Env) error {
thisHost := env.ThisHost()
thisHost := env.Bootstrap.ThisHost()
fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP)
pmuxProcConfigs := []pmuxlib.ProcessConfig{
@ -157,7 +153,7 @@ func runDaemonPmuxOnce(env *crypticnet.Env) error {
Cmd: "bash",
Args: []string{
"wait-for-ip",
env.ThisHost().Nebula.IP,
thisHost.Nebula.IP,
"bash",
"dnsmasq-entrypoint",
},
@ -170,7 +166,7 @@ func runDaemonPmuxOnce(env *crypticnet.Env) error {
Cmd: "bash",
Args: []string{
"wait-for-ip",
env.ThisHost().Nebula.IP,
thisHost.Nebula.IP,
"cryptic-net-main", "garage-entrypoint",
},

52
go-workspace/src/cmd/entrypoint/garage.go
View File

@ -2,31 +2,10 @@ package entrypoint
import (
"fmt"
"io/fs"
"log"
"os"
"strings"
"syscall"
crypticnet "cryptic-net"
"cryptic-net/garage"
)
func getGaragePeer(env *crypticnet.Env) (string, error) {
if allocs := env.ThisDaemon().Storage.Allocations; len(allocs) > 0 {
return garage.GeneratePeerAddr(env.ThisHost().Nebula.IP, allocs[0].RPCPort)
}
bootstrapPeers, err := garage.BootstrapPeerAddrs(env.Hosts)
if err != nil {
return "", err
}
return bootstrapPeers[0], nil
}
var subCmdGarageMC = subCmd{
name: "mc",
descr: "Runs the mc (minio-client) binary. The cryptic-net garage can be accessed under the `garage` alias",
@ -51,21 +30,16 @@ var subCmdGarageMC = subCmd{
env := subCmdCtx.env
apiAddr := garage.APIAddr(env)
s3APIAddr := env.ChooseGaragePeer().S3APIAddr()
if *keyID == "" || *keySecret == "" {
globalBucketCreds, err := garage.GlobalBucketAPICredentials(env)
if err != nil {
return fmt.Errorf("loading global bucket credentials: %w", err)
}
if *keyID == "" {
*keyID = globalBucketCreds.ID
*keyID = env.Bootstrap.GarageGlobalBucketS3APICredentials.ID
}
if *keySecret == "" {
*keySecret = globalBucketCreds.Secret
*keyID = env.Bootstrap.GarageGlobalBucketS3APICredentials.Secret
}
}
@ -83,7 +57,7 @@ var subCmdGarageMC = subCmd{
os.Environ(),
fmt.Sprintf(
"MC_HOST_garage=http://%s:%s@%s",
*keyID, *keySecret, apiAddr,
*keyID, *keySecret, s3APIAddr,
),
// The garage docs say this is necessary, though nothing bad
@ -111,27 +85,13 @@ var subCmdGarageCLI = subCmd{
env := subCmdCtx.env
peerAddr, err := getGaragePeer(env)
if err != nil {
return fmt.Errorf("picking peer to communicate with: %w", err)
}
rpcSecretB, err := fs.ReadFile(env.BootstrapFS, "garage/rpc-secret.txt")
if err != nil {
log.Fatalf("reading garage rpc secret bootstrap fs: %v", err)
}
rpcSecret := strings.TrimSpace(string(rpcSecretB))
var (
binPath = env.BinPath("garage")
args = append([]string{"garage"}, subCmdCtx.args...)
cliEnv = append(
os.Environ(),
"GARAGE_RPC_HOST="+peerAddr,
"GARAGE_RPC_SECRET="+rpcSecret,
"GARAGE_RPC_HOST="+env.ChooseGaragePeer().RPCAddr(),
"GARAGE_RPC_SECRET="+env.Bootstrap.GarageRPCSecret,
)
)

14
go-workspace/src/cmd/entrypoint/hosts.go
View File

@ -2,8 +2,8 @@ package entrypoint
import (
"bytes"
crypticnet "cryptic-net"
"cryptic-net/bootstrap"
bootstrap_creator "cryptic-net/bootstrap/creator"
"cryptic-net/garage"
"cryptic-net/tarutil"
"errors"
@ -73,12 +73,12 @@ var subCmdHostsAdd = subCmd{
env := subCmdCtx.env
client, err := garage.GlobalBucketAPIClient(env)
client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
nebulaHost := crypticnet.NebulaHost{
nebulaHost := bootstrap.NebulaHost{
Name: *name,
IP: *ip,
}
@ -113,7 +113,7 @@ var subCmdHostsList = subCmd{
env := subCmdCtx.env
client, err := garage.GlobalBucketAPIClient(env)
client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
@ -147,7 +147,7 @@ var subCmdHostsList = subCmd{
return fmt.Errorf("retrieving object %q from global bucket: %w", objInfo.Key, err)
}
var nebulaHost crypticnet.NebulaHost
var nebulaHost bootstrap.NebulaHost
err = yaml.NewDecoder(obj).Decode(&nebulaHost)
obj.Close()
@ -191,7 +191,7 @@ var subCmdHostsDelete = subCmd{
filePath := nebulaHostPath(*name)
client, err := garage.GlobalBucketAPIClient(env)
client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
@ -263,7 +263,7 @@ var subCmdHostsMakeBootstrap = subCmd{
return fmt.Errorf("reading admin.tgz with --admin-path of %q: %w", *adminPath, err)
}
return bootstrap.NewForHost(subCmdCtx.env, adminFS, *name, os.Stdout)
return bootstrap_creator.NewForHost(subCmdCtx.env, adminFS, *name, os.Stdout)
},
}

49
go-workspace/src/cmd/garage-entrypoint/main.go
View File

@ -2,13 +2,11 @@ package garage_entrypoint
import (
"fmt"
"io/fs"
"log"
"net"
"os"
"path/filepath"
"strconv"
"strings"
"time"
crypticnet "cryptic-net"
@ -19,24 +17,23 @@ import (
func writeChildConf(
env *crypticnet.Env,
bootstrapPeers []string,
alloc crypticnet.DaemonYmlStorageAllocation,
rpcSecret string,
) (string, error) {
if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
}
pubKey, privKey, err := garage.GeneratePeerKey(env.ThisHost().Nebula.IP, alloc.RPCPort)
thisHost := env.Bootstrap.ThisHost()
if err != nil {
return "", fmt.Errorf(
"generating node key with input %q,%d: %w",
env.ThisHost().Nebula.IP, alloc.RPCPort, err,
)
peer := garage.Peer{
IP: thisHost.Nebula.IP,
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
}
pubKey, privKey := peer.RPCPeerKey()
nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
@ -51,17 +48,17 @@ func writeChildConf(
env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
)
err = garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
MetaPath: alloc.MetaPath,
DataPath: alloc.DataPath,
RPCSecret: rpcSecret,
RPCSecret: env.Bootstrap.GarageRPCSecret,
RPCAddr: net.JoinHostPort(env.ThisHost().Nebula.IP, strconv.Itoa(alloc.RPCPort)),
APIAddr: net.JoinHostPort(env.ThisHost().Nebula.IP, strconv.Itoa(alloc.APIPort)),
WebAddr: net.JoinHostPort(env.ThisHost().Nebula.IP, strconv.Itoa(alloc.WebPort)),
RPCAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
APIAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
WebAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.WebPort)),
BootstrapPeers: bootstrapPeers,
BootstrapPeers: env.Bootstrap.GarageRPCPeerAddrs(),
})
if err != nil {
@ -73,13 +70,15 @@ func writeChildConf(
func waitForArgs(env *crypticnet.Env, bin string, binArgs ...string) []string {
thisHost := env.Bootstrap.ThisHost()
var args []string
for _, alloc := range env.ThisDaemon().Storage.Allocations {
args = append(
args,
"wait-for",
net.JoinHostPort(env.ThisHost().Nebula.IP, strconv.Itoa(alloc.RPCPort)),
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
"--",
)
}
@ -98,25 +97,11 @@ func Main() {
log.Fatalf("reading envvars: %v", err)
}
bootstrapPeers, err := garage.BootstrapPeerAddrs(env.Hosts)
if err != nil {
log.Fatalf("generating set of bootstrap peers: %v", err)
}
rpcSecretB, err := fs.ReadFile(env.BootstrapFS, "garage/rpc-secret.txt")
if err != nil {
log.Fatalf("reading garage rpc secret bootstrap fs: %v", err)
}
rpcSecret := strings.TrimSpace(string(rpcSecretB))
var pmuxProcConfigs []pmuxlib.ProcessConfig
for _, alloc := range env.ThisDaemon().Storage.Allocations {
childConfPath, err := writeChildConf(env, bootstrapPeers, alloc, rpcSecret)
childConfPath, err := writeChildConf(env, alloc)
if err != nil {
log.Fatalf("writing child config file for alloc %+v: %v", alloc, err)

29
go-workspace/src/cmd/garage-layout-diff/main.go
View File

@ -148,29 +148,30 @@ func readCurrNodes(r io.Reader) (clusterNodes, int, error) {
return currNodes, version, nil
}
func readExpNodes(env *crypticnet.Env) (clusterNodes, error) {
func readExpNodes(env *crypticnet.Env) clusterNodes {
thisHost := env.Bootstrap.ThisHost()
var expNodes clusterNodes
for _, alloc := range env.ThisDaemon().Storage.Allocations {
id, err := garage.GeneratePeerID(env.ThisHost().Nebula.IP, alloc.RPCPort)
if err != nil {
return nil, fmt.Errorf(
"generating peer id for ip:%q port:%d: %w",
env.ThisHost().Nebula.IP, alloc.RPCPort, err,
)
peer := garage.Peer{
IP: thisHost.Nebula.IP,
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
}
id := peer.RPCPeerID()
expNodes = append(expNodes, clusterNode{
ID: id,
Zone: env.ThisHost().Name,
Zone: env.Bootstrap.HostName,
Capacity: alloc.Capacity / 100,
})
}
return expNodes, nil
return expNodes
}
// NOTE: The id formatting for currNodes and expNodes is different; expNodes has
@ -232,18 +233,14 @@ func Main() {
for _, node := range currNodes {
if env.ThisHost().Name != node.Zone {
if env.Bootstrap.HostName != node.Zone {
continue
}
thisCurrNodes = append(thisCurrNodes, node)
}
expNodes, err := readExpNodes(env)
if err != nil {
panic(fmt.Errorf("reading expected layout from environment: %w", err))
}
expNodes := readExpNodes(env)
lines := diff(thisCurrNodes, expNodes)

9
go-workspace/src/cmd/garage-peer-keygen/main.go
View File

@ -42,12 +42,13 @@ func Main() {
panic("The arguments -ip, -port, and -danger are required")
}
pubKey, privKey, err := garage.GeneratePeerKey(*ip, *port)
if err != nil {
panic(fmt.Errorf("GeneratePeerKey returned: %w", err))
peer := garage.Peer{
IP: *ip,
RPCPort: *port,
}
pubKey, privKey := peer.RPCPeerKey()
fmt.Fprintln(os.Stdout, hex.EncodeToString(pubKey))
if *outPub != "" {

18
go-workspace/src/cmd/garage-update-global-bucket/main.go
View File

@ -3,6 +3,7 @@ package garage_update_global_bucket
import (
"bytes"
crypticnet "cryptic-net"
"cryptic-net/bootstrap"
"cryptic-net/garage"
"fmt"
"log"
@ -16,12 +17,15 @@ func updateGlobalBucket(env *crypticnet.Env) error {
ctx := env.Context
client, err := garage.GlobalBucketAPIClient(env)
client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
filePath := filepath.Join("garage/hosts", env.ThisHost().Name+".yml")
filePath := filepath.Join(
"garage/hosts",
env.Bootstrap.HostName+".yml",
)
daemon := env.ThisDaemon()
@ -41,14 +45,14 @@ func updateGlobalBucket(env *crypticnet.Env) error {
return nil
}
var garageHost crypticnet.GarageHost
var garageHost bootstrap.GarageHost
for _, alloc := range daemon.Storage.Allocations {
garageHostInstance := crypticnet.GarageHostInstance{
APIPort: alloc.APIPort,
RPCPort: alloc.RPCPort,
WebPort: alloc.WebPort,
garageHostInstance := bootstrap.GarageHostInstance{
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
WebPort: alloc.WebPort,
}
garageHost.Instances = append(garageHost.Instances, garageHostInstance)

27
go-workspace/src/cmd/nebula-entrypoint/main.go
View File

@ -2,8 +2,6 @@ package nebula_entrypoint
import (
"cryptic-net/yamlutil"
"fmt"
"io/fs"
"log"
"net"
"path/filepath"
@ -27,7 +25,7 @@ func Main() {
staticHostMap = map[string][]string{}
)
for _, host := range env.Hosts {
for _, host := range env.Bootstrap.Hosts {
if host.Nebula.PublicAddr == "" {
continue
@ -37,26 +35,11 @@ func Main() {
staticHostMap[host.Nebula.IP] = []string{host.Nebula.PublicAddr}
}
readCertFile := func(name string) string {
if err != nil {
return ""
}
path := filepath.Join("nebula", "certs", name)
var b []byte
if b, err = fs.ReadFile(env.BootstrapFS, path); err != nil {
err = fmt.Errorf("reading %q from bootstrap fs: %w", path, err)
}
return string(b)
}
config := map[string]interface{}{
"pki": map[string]string{
"ca": readCertFile("ca.crt"),
"cert": readCertFile("host.crt"),
"key": readCertFile("host.key"),
"ca": env.Bootstrap.NebulaCertsCACert,
"cert": env.Bootstrap.NebulaCertsHostCert,
"key": env.Bootstrap.NebulaCertsHostKey,
},
"static_host_map": staticHostMap,
"punchy": map[string]bool{
@ -110,7 +93,7 @@ func Main() {
firewallInbound = append(
firewallInbound,
crypticnet.ConfigFirewallRule{
Port: strconv.Itoa(alloc.APIPort),
Port: strconv.Itoa(alloc.S3APIPort),
Proto: "tcp",
Host: "any",
},

4
go-workspace/src/cmd/nebula-update-global-bucket/main.go
View File

@ -16,14 +16,14 @@ func updateGlobalBucket(env *crypticnet.Env) error {
ctx := env.Context
client, err := garage.GlobalBucketAPIClient(env)
client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
daemon := env.ThisDaemon()
host := env.ThisHost()
host := env.Bootstrap.ThisHost()
host.Nebula.Name = host.Name
host.Nebula.PublicAddr = daemon.VPN.PublicAddr

12
go-workspace/src/daemon_yml.go
View File

@ -28,12 +28,12 @@ type ConfigFirewallRule struct {
// DaemonYmlStorageAllocation describes the structure of each storage allocation
// within the daemon.yml file.
type DaemonYmlStorageAllocation struct {
DataPath string `yaml:"data_path"`
MetaPath string `yaml:"meta_path"`
Capacity int `yaml:"capacity"`
APIPort int `yaml:"api_port"`
RPCPort int `yaml:"rpc_port"`
WebPort int `yaml:"web_port"`
DataPath string `yaml:"data_path"`
MetaPath string `yaml:"meta_path"`
Capacity int `yaml:"capacity"`
S3APIPort int `yaml:"api_port"` // TODO fix field name here
RPCPort int `yaml:"rpc_port"`
WebPort int `yaml:"web_port"`
}
// DaemonYml describes the structure of the daemon.yml file.

36
go-workspace/src/env.go
View File

@ -2,7 +2,7 @@ package crypticnet
import (
"context"
"cryptic-net/tarutil"
"cryptic-net/bootstrap"
"cryptic-net/yamlutil"
"errors"
"fmt"
@ -37,9 +37,7 @@ type Env struct {
// If NewEnv is called with bootstrapOptional, and a bootstrap file is not
// found, then these fields will not be set.
BootstrapPath string
BootstrapFS fs.FS
Hosts map[string]Host
HostName string
Bootstrap bootstrap.Bootstrap
thisDaemon DaemonYml
thisDaemonOnce sync.Once
@ -118,32 +116,13 @@ func (e *Env) DataDirBootstrapPath() string {
// and all derived fields based on that.
func (e *Env) LoadBootstrap(path string) error {
var (
err error
var err error
// load all values into temp variables before setting the fields on Env,
// so we don't leave it in an inconsistent state.
bootstrapFS fs.FS
hosts map[string]Host
hostNameB []byte
)
if bootstrapFS, err = tarutil.FSFromTGZFile(path); err != nil {
return fmt.Errorf("reading bootstrap file at %q: %w", e.BootstrapPath, err)
}
if hosts, err = LoadHosts(bootstrapFS); err != nil {
return fmt.Errorf("loading hosts info from bootstrap fs: %w", err)
}
if hostNameB, err = fs.ReadFile(bootstrapFS, "hostname"); err != nil {
return fmt.Errorf("loading hostname from bootstrap fs: %w", err)
if e.Bootstrap, err = bootstrap.FromFile(path); err != nil {
return fmt.Errorf("parsing bootstrap.tgz at %q: %w", path, err)
}
e.BootstrapPath = path
e.BootstrapFS = bootstrapFS
e.Hosts = hosts
e.HostName = string(hostNameB)
return nil
}
@ -219,11 +198,6 @@ func (e *Env) init(bootstrapOptional bool) error {
return nil
}
// ThisHost is a shortcut for returning env.Hosts[env.HostName].
func (e *Env) ThisHost() Host {
return e.Hosts[e.HostName]
}
// ToMap returns the Env as a map of key/value strings. If this map is set into
// a process's environment, then that process can read it back using ReadEnv.
func (e *Env) ToMap() map[string]string {

42
go-workspace/src/garage.go Normal file
View File

@ -0,0 +1,42 @@
package crypticnet
import (
"cryptic-net/garage"
"fmt"
)
// ChooseGaragePeer returns a Peer for a garage instance from the network. It
// will prefer a garage instance on this particular host, if there is one, but
// will otherwise return a random endpoint.
func (env *Env) ChooseGaragePeer() garage.Peer {
if allocs := env.ThisDaemon().Storage.Allocations; len(allocs) > 0 {
return garage.Peer{
IP: env.Bootstrap.ThisHost().Nebula.IP,
RPCPort: allocs[0].RPCPort,
S3APIPort: allocs[0].S3APIPort,
}
}
for _, peer := range env.Bootstrap.GaragePeers() {
return peer
}
panic("no garage instances configured")
}
// GlobalBucketS3APIClient returns an S3 client pre-configured with access to
// the global bucket.
func (env *Env) GlobalBucketS3APIClient() (garage.S3APIClient, error) {
addr := env.ChooseGaragePeer().S3APIAddr()
creds := env.Bootstrap.GarageGlobalBucketS3APICredentials
client, err := garage.NewS3APIClient(addr, creds)
if err != nil {
return nil, fmt.Errorf("connecting to garage S3 API At %q: %w", addr, err)
}
return client, err
}

74
go-workspace/src/garage/client.go
View File

@ -1,12 +1,7 @@
package garage
import (
crypticnet "cryptic-net"
"cryptic-net/yamlutil"
"errors"
"fmt"
"net"
"strconv"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
@ -19,74 +14,21 @@ func IsKeyNotFound(err error) bool {
return errors.As(err, &mErr) && mErr.Code == "NoSuchKey"
}
// APICredentials describe data fields necessary for authenticating with a
// garage api endpoint.
type APICredentials struct {
// S3APIClient is a client used to interact with garage's S3 API.
type S3APIClient = *minio.Client
// S3APICredentials describe data fields necessary for authenticating with a
// garage S3 API endpoint.
type S3APICredentials struct {
ID string `yaml:"id"`
Secret string `yaml:"secret"`
}
// GlobalBucketAPICredentials returns APICredentials for the global bucket.
func GlobalBucketAPICredentials(env *crypticnet.Env) (APICredentials, error) {
const path = "garage/priv/global-bucket-key.yml"
var creds APICredentials
if err := yamlutil.LoadYamlFSFile(&creds, env.BootstrapFS, path); err != nil {
return APICredentials{}, fmt.Errorf("loading %q from bootstrap fs: %w", path, err)
}
return creds, nil
}
// APIAddr returns the network address of a garage api endpoint in the network.
// It will prefer an endpoint on this particular host, if there is one, but will
// otherwise return a random endpoint.
func APIAddr(env *crypticnet.Env) string {
if allocs := env.ThisDaemon().Storage.Allocations; len(allocs) > 0 {
return net.JoinHostPort(
env.ThisHost().Nebula.IP,
strconv.Itoa(allocs[0].APIPort),
)
}
for _, host := range env.Hosts {
if host.Garage == nil || len(host.Garage.Instances) == 0 {
continue
}
return net.JoinHostPort(
host.Nebula.IP,
strconv.Itoa(host.Garage.Instances[0].APIPort),
)
}
panic("no garage instances configured")
}
// APIClient returns a minio client configured to use the given garage API
// NewS3APIClient returns a minio client configured to use the given garage S3 API
// endpoint.
func APIClient(addr string, creds APICredentials) (*minio.Client, error) {
func NewS3APIClient(addr string, creds S3APICredentials) (S3APIClient, error) {
return minio.New(addr, &minio.Options{
Creds: credentials.NewStaticV4(creds.ID, creds.Secret, ""),
Region: Region,
})
}
// GlobalBucketAPIClient returns a minio client pre-configured with access to
// the global bucket.
func GlobalBucketAPIClient(env *crypticnet.Env) (*minio.Client, error) {
creds, err := GlobalBucketAPICredentials(env)
if err != nil {
return nil, fmt.Errorf("loading global bucket credentials: %w", err)
}
addr := APIAddr(env)
return APIClient(addr, creds)
}

87
go-workspace/src/garage/garage.go
View File

@ -2,15 +2,6 @@
// setting up garage configs, processes, and deployments.
package garage
import (
crypticnet "cryptic-net"
"crypto/ed25519"
"encoding/hex"
"fmt"
"net"
"strconv"
)
const (
// Region is the region which garage is configured with.
@ -20,81 +11,3 @@ const (
// accessible to all hosts in the network.
GlobalBucket = "cryptic-net-global"
)
// GeneratePeerKey deterministically generates a public/private keys which can
// be used as a garage node key.
//
// DANGER: This function will deterministically produce public/private keys
// given some arbitrary input. This is NEVER what you want. It's only being used
// in cryptic-net for a very specific purpose for which I think it's ok and is
// very necessary, and people are probably _still_ going to yell at me.
//
func GeneratePeerKey(ip string, port int) (pubKey, privKey []byte, err error) {
input := []byte(net.JoinHostPort(ip, strconv.Itoa(port)))
// Append the length of the input to the input, so that the input "foo"
// doesn't generate the same key as the input "foofoo".
input = strconv.AppendInt(input, int64(len(input)), 10)
return ed25519.GenerateKey(NewInfiniteReader(input))
}
// GeneratePeerID generates the peer id for the given peer.
//
// DANGER: See warning on GenerateNodeKey.
func GeneratePeerID(ip string, port int) (string, error) {
peerNodeKeyPub, _, err := GeneratePeerKey(ip, port)
if err != nil {
return "", err
}
return hex.EncodeToString(peerNodeKeyPub), nil
}
// GeneratePeerAddr generates the peer address (e.g. "id@ip:port") for the
// given peer.
//
// DANGER: See warning on GenerateNodeKey.
func GeneratePeerAddr(ip string, port int) (string, error) {
id, err := GeneratePeerID(ip, port)
if err != nil {
return "", fmt.Errorf("generating peer id: %w", err)
}
return fmt.Sprintf("%s@%s", id, net.JoinHostPort(ip, strconv.Itoa(port))), nil
}
// BootstrapPeerAddrs generates the list of bootstrap peer strings based on the
// bootstrap hosts.
func BootstrapPeerAddrs(hosts map[string]crypticnet.Host) ([]string, error) {
var peers []string
for _, host := range hosts {
if host.Garage == nil {
continue
}
for _, instance := range host.Garage.Instances {
peer, err := GeneratePeerAddr(host.Nebula.IP, instance.RPCPort)
if err != nil {
return nil, fmt.Errorf(
"generating peer address with input %q,%d: %w",
host.Nebula.IP, instance.RPCPort, err,
)
}
peers = append(peers, peer)
}
}
return peers, nil
}

66
go-workspace/src/garage/peer.go Normal file
View File

@ -0,0 +1,66 @@
package garage
import (
"crypto/ed25519"
"encoding/hex"
"fmt"
"net"
"strconv"
)
// Peer describes all information necessary to connect to a given garage node.
type Peer struct {
IP string
RPCPort int
S3APIPort int
}
// RPCPeerKey deterministically generates a public/private keys which can
// be used as a garage node key.
//
// DANGER: This function will deterministically produce public/private keys
// given some arbitrary input. This is NEVER what you want. It's only being used
// in cryptic-net for a very specific purpose for which I think it's ok and is
// very necessary, and people are probably _still_ going to yell at me.
//
func (p Peer) RPCPeerKey() (pubKey, privKey []byte) {
input := []byte(net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort)))
// Append the length of the input to the input, so that the input "foo"
// doesn't generate the same key as the input "foofoo".
input = strconv.AppendInt(input, int64(len(input)), 10)
pubKey, privKey, err := ed25519.GenerateKey(NewInfiniteReader(input))
if err != nil {
panic(err)
}
return pubKey, privKey
}
// RPCPeerID returns the peer ID of the garage node for use in communicating
// over RPC.
//
// DANGER: See warning on RPCPeerKey.
func (p Peer) RPCPeerID() string {
pubKey, _ := p.RPCPeerKey()
return hex.EncodeToString(pubKey)
}
// RPCAddr returns the address of the peer's RPC port.
func (p Peer) RPCAddr() string {
return net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort))
}
// RPCPeerAddr returns the full peer address (e.g. "id@ip:port") of the garage
// node for use in communicating over RPC.
//
// DANGER: See warning on RPCPeerKey.
func (p Peer) RPCPeerAddr() string {
return fmt.Sprintf("%s@%s", p.RPCPeerID(), p.RPCAddr())
}
// S3APIAddr returns the address of the peer's S3 API port.
func (p Peer) S3APIAddr() string {
return net.JoinHostPort(p.IP, strconv.Itoa(p.S3APIPort))
}

4
go-workspace/src/nebula/nebula.go
View File

@ -3,7 +3,7 @@
package nebula
import (
crypticnet "cryptic-net"
"cryptic-net/bootstrap"
"crypto/ed25519"
"crypto/rand"
"fmt"
@ -43,7 +43,7 @@ type CACert struct {
// NewHostCert generates a new key/cert for a nebula host using the CA key
// which will be found in the adminFS.
func NewHostCert(
adminFS fs.FS, host crypticnet.NebulaHost,
adminFS fs.FS, host bootstrap.NebulaHost,
) (
HostCert, error,
) {

15
go-workspace/src/tarutil/tarutil.go
View File

@ -7,12 +7,11 @@ import (
"fmt"
"io"
"io/fs"
"os"
"github.com/nlepage/go-tarfs"
)
// FSFromTGZFile returns a FS instance which will read the contents of a tgz
// FSFromReader returns a FS instance which will read the contents of a tgz
// file from the given Reader.
func FSFromReader(r io.Reader) (fs.FS, error) {
gf, err := gzip.NewReader(r)
@ -23,15 +22,3 @@ func FSFromReader(r io.Reader) (fs.FS, error) {
return tarfs.New(gf)
}
// FSFromTGZFile returns a FS instance which will read the contents of a tgz
// file.
func FSFromTGZFile(path string) (fs.FS, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("opening file: %w", err)
}
defer f.Close()
return FSFromReader(f)
}