More big refactoring leading up to network creation

The `bootstrap/creator` package is gone, almost as quickly as it
arrived. The `Bootstrap` type is now able to write its own tgz file, and
the two places where bootstrap files are being created pull the data
down to do so and create the `Bootstrap` structs directly.

The structure of the bootstrap file itself has been changed, now there's
just a single `hosts` directory which contains files which are yaml
encodings of the `Host` type, rather than having it be split into
`nebula` and `garage` directories. This makes creating bootstrap files a
lot easier.

go-workspace/src/bootstrap/bootstrap.go

@@ -5,16 +5,21 @@ import (
 	"cryptic-net/garage"
 	"cryptic-net/tarutil"
 	"cryptic-net/yamlutil"
+	"crypto/sha512"
 	"fmt"
 	"io"
 	"io/fs"
 	"os"
+	"path/filepath"
+	"sort"
 	"strings"
+
+	"gopkg.in/yaml.v3"
 )
 
 // Paths within the bootstrap FS which for general data.
 const (
-	HostNamePath = "hostname"
+	hostNamePath = "hostname"
 )
 
 // Bootstrap is used for accessing all information contained within a
@@ -32,14 +37,6 @@ type Bootstrap struct {
 
 	GarageRPCSecret                    string
 	GarageGlobalBucketS3APICredentials garage.S3APICredentials
-
-	// Hash is a determinstic hash of the contents of the bootstrap file. This
-	// will be populated when parsing a Bootstrap from a bootstrap.tgz, but will
-	// be ignored when creating a new bootstrap.tgz.
-	Hash []byte
-
-	// DEPRECATED do not use
-	FS fs.FS
 }
 
 // FromFS loads a Boostrap instance from the given fs.FS, which presumably
@@ -51,8 +48,6 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
 		err error
 	)
 
-	b.FS = bootstrapFS
-
 	if b.Hosts, err = loadHosts(bootstrapFS); err != nil {
 		return Bootstrap{}, fmt.Errorf("loading hosts info from fs: %w", err)
 	}
@@ -60,7 +55,7 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
 	if err = yamlutil.LoadYamlFSFile(
 		&b.GarageGlobalBucketS3APICredentials,
 		bootstrapFS,
-		GarageGlobalBucketKeyYmlPath,
+		garageGlobalBucketKeyYmlPath,
 	); err != nil {
 		return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", b.GarageGlobalBucketS3APICredentials, err)
 	}
@@ -69,11 +64,11 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
 		into *string
 		path string
 	}{
-		{&b.HostName, HostNamePath},
+		{&b.HostName, hostNamePath},
-		{&b.NebulaCertsCACert, NebulaCertsCACertPath},
+		{&b.NebulaCertsCACert, nebulaCertsCACertPath},
-		{&b.NebulaCertsHostCert, NebulaCertsHostCertPath},
+		{&b.NebulaCertsHostCert, nebulaCertsHostCertPath},
-		{&b.NebulaCertsHostKey, NebulaCertsHostKeyPath},
+		{&b.NebulaCertsHostKey, nebulaCertsHostKeyPath},
-		{&b.GarageRPCSecret, GarageRPCSecretPath},
+		{&b.GarageRPCSecret, garageRPCSecretPath},
 	}
 
 	for _, f := range filesToLoadAsString {
@@ -87,10 +82,6 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
 	// TODO confirm if this is necessary
 	b.GarageRPCSecret = strings.TrimSpace(b.GarageRPCSecret)
 
-	if b.Hash, err = fs.ReadFile(bootstrapFS, tarutil.HashBinPath); err != nil {
-		return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", tarutil.HashBinPath, err)
-	}
-
 	return b, nil
 }
 
@@ -117,6 +108,48 @@ func FromFile(path string) (Bootstrap, error) {
 	return FromReader(f)
 }
 
+// WriteTo writes the Bootstrap as a new bootstrap.tgz to the given io.Writer.
+func (b Bootstrap) WriteTo(into io.Writer) error {
+
+	w := tarutil.NewTGZWriter(into)
+
+	filesToWriteAsString := []struct {
+		value string
+		path  string
+	}{
+		{b.HostName, hostNamePath},
+		{b.NebulaCertsCACert, nebulaCertsCACertPath},
+		{b.NebulaCertsHostCert, nebulaCertsHostCertPath},
+		{b.NebulaCertsHostKey, nebulaCertsHostKeyPath},
+		{b.GarageRPCSecret, garageRPCSecretPath},
+	}
+
+	for _, f := range filesToWriteAsString {
+		w.WriteFileBytes(f.path, []byte(f.value))
+	}
+
+	garageGlobalBucketKeyB, err := yaml.Marshal(b.GarageGlobalBucketS3APICredentials)
+	if err != nil {
+		return fmt.Errorf("yaml encoding garage global bucket creds: %w", err)
+	}
+
+	w.WriteFileBytes(garageGlobalBucketKeyYmlPath, garageGlobalBucketKeyB)
+
+	for _, host := range b.Hosts {
+
+		hostB, err := yaml.Marshal(host)
+		if err != nil {
+			return fmt.Errorf("yaml encoding host %#v: %w", host, err)
+		}
+
+		path := filepath.Join(hostsDirPath, host.Name+".yml")
+
+		w.WriteFileBytes(path, hostB)
+	}
+
+	return w.Close()
+}
+
 // ThisHost is a shortcut for b.Hosts[b.HostName], but will panic if the
 // HostName isn't found in the Hosts map.
 func (b Bootstrap) ThisHost() Host {
@@ -128,3 +161,38 @@ func (b Bootstrap) ThisHost() Host {
 
 	return host
 }
+
+// Hash returns a deterministic hash of the given hosts map.
+func HostsHash(hostsMap map[string]Host) ([]byte, error) {
+
+	hosts := make([]Host, 0, len(hostsMap))
+	for _, host := range hostsMap {
+		hosts = append(hosts, host)
+	}
+
+	sort.Slice(hosts, func(i, j int) bool { return hosts[i].Name < hosts[j].Name })
+
+	h := sha512.New()
+
+	if err := yaml.NewEncoder(h).Encode(hosts); err != nil {
+		return nil, err
+	}
+
+	return h.Sum(nil), nil
+}
+
+// WithHosts returns a copy of the Bootstrap with the given set of Hosts applied
+// to it. It will _not_ overwrite the Host for _this_ host, however.
+func (b Bootstrap) WithHosts(hosts map[string]Host) Bootstrap {
+
+	hostsCopy := make(map[string]Host, len(hosts))
+
+	for name, host := range hosts {
+		hostsCopy[name] = host
+	}
+
+	hostsCopy[b.HostName] = b.ThisHost()
+
+	b.Hosts = hostsCopy
+	return b
+}

go-workspace/src/bootstrap/creator/creator.go

@@ -1,110 +0,0 @@
-// Package creator is responsible for creating bootstrap files. It exists
-// separately from the main bootstrap package in order to prevent import loops
-// due to its use of crypticnet.Env.
-package creator
-
-import (
-	"context"
-	crypticnet "cryptic-net"
-	"cryptic-net/bootstrap"
-	"cryptic-net/nebula"
-	"cryptic-net/tarutil"
-	"fmt"
-	"io"
-	"io/fs"
-)
-
-func newBootstrap(
-	ctx context.Context,
-	into io.Writer,
-	hostname provider,
-	nebulaCertsCACert provider,
-	nebulaCertsHostCert provider,
-	nebulaCertsHostKey provider,
-	nebulaHosts provider,
-	garageRPCSecret provider,
-	garageGlobalBucketKey provider,
-	garageHosts provider,
-) error {
-
-	pairs := []struct {
-		path     string
-		provider provider
-	}{
-		{bootstrap.HostNamePath, hostname},
-		{bootstrap.NebulaCertsCACertPath, nebulaCertsCACert},
-		{bootstrap.NebulaCertsHostCertPath, nebulaCertsHostCert},
-		{bootstrap.NebulaCertsHostKeyPath, nebulaCertsHostKey},
-		{bootstrap.NebulaHostsDirPath, nebulaHosts},
-		{bootstrap.GarageRPCSecretPath, garageRPCSecret},
-		{bootstrap.GarageGlobalBucketKeyYmlPath, garageGlobalBucketKey},
-		{bootstrap.GarageHostsDirPath, garageHosts},
-	}
-
-	w := tarutil.NewTGZWriter(into)
-
-	for _, pair := range pairs {
-
-		if err := pair.provider(ctx, w, pair.path); err != nil {
-			return fmt.Errorf("populating %q in new bootstrap: %w", pair.path, err)
-		}
-	}
-
-	return w.Close()
-}
-
-// NewForThisHost generates a new bootstrap file for the current host, based on
-// the existing environment as well as data in garage.
-func NewForThisHost(env *crypticnet.Env, into io.Writer) error {
-
-	client, err := env.GlobalBucketS3APIClient()
-	if err != nil {
-		return fmt.Errorf("creating client for global bucket: %w", err)
-	}
-
-	return newBootstrap(
-		env.Context,
-		into,
-		provideFromFS(env.Bootstrap.FS), // hostname
-		provideFromFS(env.Bootstrap.FS), // nebulaCertsCACert
-		provideFromFS(env.Bootstrap.FS), // nebulaCertsHostCert
-		provideFromFS(env.Bootstrap.FS), // nebulaCertsHostKey
-		provideDirFromGarage(client),    // nebulaHosts
-		provideFromFS(env.Bootstrap.FS), // garageRPCSecret
-		provideFromFS(env.Bootstrap.FS), // garageGlobalBucketKey
-		provideDirFromGarage(client),    // garageHosts
-	)
-}
-
-// NewForHost generates a new bootstrap file for an arbitrary host, based on the
-// given admin file's FS and data in garage.
-func NewForHost(env *crypticnet.Env, adminFS fs.FS, name string, into io.Writer) error {
-
-	host, ok := env.Bootstrap.Hosts[name]
-	if !ok {
-		return fmt.Errorf("unknown host %q, make sure host entry has been created", name)
-	}
-
-	client, err := env.GlobalBucketS3APIClient()
-	if err != nil {
-		return fmt.Errorf("creating client for global bucket: %w", err)
-	}
-
-	nebulaHostCert, err := nebula.NewHostCert(adminFS, host.Nebula)
-	if err != nil {
-		return fmt.Errorf("creating new nebula host key/cert: %w", err)
-	}
-
-	return newBootstrap(
-		env.Context,
-		into,
-		provideFromBytes([]byte(name)),          // hostname
-		provideFromBytes(nebulaHostCert.CACert), // nebulaCertsCACert
-		provideFromBytes(nebulaHostCert.HostCert), // nebulaCertsHostCert
-		provideFromBytes(nebulaHostCert.HostKey),  // nebulaCertsHostKey
-		provideDirFromGarage(client),              // nebulaHosts
-		provideFromFS(adminFS),                    // garageRPCSecret
-		provideFromFS(adminFS),                    // garageGlobalBucketKey
-		provideDirFromGarage(client),              // garageHosts
-	)
-}

go-workspace/src/bootstrap/creator/provider.go

@@ -1,121 +0,0 @@
-package creator
-
-import (
-	"context"
-	"cryptic-net/garage"
-	"cryptic-net/tarutil"
-	"fmt"
-	"io/fs"
-
-	"github.com/minio/minio-go/v7"
-)
-
-// provider is a function which will populate the given filePath into the given
-// TGZWriter. The path may be a file or a directory.
-type provider func(context.Context, *tarutil.TGZWriter, string) error
-
-func provideFromBytes(body []byte) provider {
-
-	return func(
-		ctx context.Context,
-		w *tarutil.TGZWriter,
-		filePath string,
-	) error {
-
-		w.WriteFileBytes(filePath, body)
-		return nil
-	}
-}
-
-func provideFromFS(srcFS fs.FS) provider {
-
-	return func(
-		ctx context.Context,
-		w *tarutil.TGZWriter,
-		filePath string,
-	) error {
-
-		return w.CopyFileFromFS(filePath, srcFS)
-	}
-}
-
-func provideDirFromFS(srcFS fs.FS) provider {
-
-	return func(
-		ctx context.Context,
-		w *tarutil.TGZWriter,
-		dirPath string,
-	) error {
-
-		return fs.WalkDir(
-			srcFS, dirPath,
-			func(filePath string, dirEntry fs.DirEntry, err error) error {
-
-				if err != nil {
-					return err
-
-				} else if dirEntry.IsDir() {
-					return nil
-
-				} else if err := w.CopyFileFromFS(filePath, srcFS); err != nil {
-					return fmt.Errorf("copying file %q: %w", filePath, err)
-				}
-
-				return nil
-			},
-		)
-
-	}
-}
-
-// TODO it'd be great if we could wrap a minio.Client into an fs.FS. That would
-// get rid of a weird dependency in this package, and clean up this code a ton.
-func provideDirFromGarage(client *minio.Client) provider {
-
-	return func(
-		ctx context.Context,
-		w *tarutil.TGZWriter,
-		dirPath string,
-	) error {
-
-		objInfoCh := client.ListObjects(
-			ctx, garage.GlobalBucket,
-			minio.ListObjectsOptions{
-				Prefix:    dirPath,
-				Recursive: true,
-			},
-		)
-
-		for objInfo := range objInfoCh {
-
-			if objInfo.Err != nil {
-				return fmt.Errorf("listing objects: %w", objInfo.Err)
-			}
-
-			obj, err := client.GetObject(
-				ctx, garage.GlobalBucket, objInfo.Key, minio.GetObjectOptions{},
-			)
-
-			if err != nil {
-				return fmt.Errorf(
-					"retrieving object %q from global bucket: %w",
-					objInfo.Key, err,
-				)
-			}
-
-			objStat, err := obj.Stat()
-			if err != nil {
-				obj.Close()
-				return fmt.Errorf(
-					"stating object %q from global bucket: %w",
-					objInfo.Key, err,
-				)
-			}
-
-			w.WriteFile(objInfo.Key, objStat.Size, obj)
-			obj.Close()
-		}
-
-		return nil
-	}
-}

go-workspace/src/bootstrap/garage.go

@@ -6,9 +6,8 @@ import (
 
 // Paths within the bootstrap FS related to garage.
 const (
-	GarageGlobalBucketKeyYmlPath = "garage/global-bucket-key.yml"
+	garageGlobalBucketKeyYmlPath = "garage/global-bucket-key.yml"
-	GarageRPCSecretPath          = "garage/rpc-secret.txt"
+	garageRPCSecretPath          = "garage/rpc-secret.txt"
-	GarageHostsDirPath           = "garage/hosts"
 )
 
 // GaragePeers returns a Peer for each known garage instance in the network.

go-workspace/src/bootstrap/garage_global_bucket.go

@@ -0,0 +1,111 @@
+package bootstrap
+
+import (
+	"bytes"
+	"context"
+	"cryptic-net/garage"
+	"fmt"
+	"log"
+	"path/filepath"
+
+	"github.com/minio/minio-go/v7"
+	"gopkg.in/yaml.v3"
+)
+
+// Paths within garage's global bucket
+const (
+	garageGlobalBucketBootstrapHostsDirPath = "bootstrap/hosts"
+)
+
+// PutGarageBoostrapHost places the <hostname>.yml file for the given host into
+// garage so that other hosts are able to see relevant configuration for it.
+//
+// The given client should be for the global bucket.
+func PutGarageBoostrapHost(
+	ctx context.Context, client garage.S3APIClient, host Host,
+) error {
+
+	buf := new(bytes.Buffer)
+
+	if err := yaml.NewEncoder(buf).Encode(host); err != nil {
+		log.Fatalf("yaml encoding host data: %v", err)
+	}
+
+	filePath := filepath.Join(garageGlobalBucketBootstrapHostsDirPath, host.Name+".yml")
+
+	_, err := client.PutObject(
+		ctx, garage.GlobalBucket, filePath, buf, int64(buf.Len()),
+		minio.PutObjectOptions{},
+	)
+
+	if err != nil {
+		return fmt.Errorf("writing to %q in global bucket: %w", filePath, err)
+	}
+
+	return nil
+}
+
+// RemoveGarageBootstrapHost removes the <hostname>.yml for the given host from
+// garage.
+//
+// The given client should be for the global bucket.
+func RemoveGarageBootstrapHost(
+	ctx context.Context, client garage.S3APIClient, hostName string,
+) error {
+
+	filePath := filepath.Join(garageGlobalBucketBootstrapHostsDirPath, hostName+".yml")
+
+	return client.RemoveObject(
+		ctx, garage.GlobalBucket, filePath,
+		minio.RemoveObjectOptions{},
+	)
+}
+
+// GetGarageBootstrapHosts loads the <hostname>.yml file for all hosts stored in
+// garage.
+//
+// The given client should be for the global bucket.
+func GetGarageBootstrapHosts(
+	ctx context.Context, client garage.S3APIClient,
+) (
+	map[string]Host, error,
+) {
+
+	hosts := map[string]Host{}
+
+	objInfoCh := client.ListObjects(
+		ctx, garage.GlobalBucket,
+		minio.ListObjectsOptions{
+			Prefix:    garageGlobalBucketBootstrapHostsDirPath,
+			Recursive: true,
+		},
+	)
+
+	for objInfo := range objInfoCh {
+
+		if objInfo.Err != nil {
+			return nil, fmt.Errorf("listing objects: %w", objInfo.Err)
+		}
+
+		obj, err := client.GetObject(
+			ctx, garage.GlobalBucket, objInfo.Key, minio.GetObjectOptions{},
+		)
+
+		if err != nil {
+			return nil, fmt.Errorf("retrieving object %q: %w", objInfo.Key, err)
+		}
+
+		var host Host
+
+		err = yaml.NewDecoder(obj).Decode(&host)
+		obj.Close()
+
+		if err != nil {
+			return nil, fmt.Errorf("yaml decoding object %q: %w", objInfo.Key, err)
+		}
+
+		hosts[host.Name] = host
+	}
+
+	return hosts, nil
+}

go-workspace/src/bootstrap/hosts.go

@@ -10,21 +10,26 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-// NebulaHost describes the contents of a `./nebula/hosts/<hostname>.yml` file.
+const (
+	hostsDirPath = "hosts"
+)
+
+// NebulaHost describes the nebula configuration of a Host which is relevant for
+// other hosts to know.
 type NebulaHost struct {
-	Name       string `yaml:"name"`
 	IP         string `yaml:"ip"`
 	PublicAddr string `yaml:"public_addr,omitempty"`
 }
 
-// GarageHostInstance describes a single garage instance running on a host.
+// GarageHost describes a single garage instance in the GarageHost.
 type GarageHostInstance struct {
 	RPCPort   int `yaml:"rpc_port"`
 	S3APIPort int `yaml:"s3_api_port"`
 	WebPort   int `yaml:"web_port"`
 }
 
-// GarageHost describes the contents of a `./garage/hosts/<hostname>.yml` file.
+// GarageHost describes the garage configuration of a Host which is relevant for
+// other hosts to know.
 type GarageHost struct {
 	Instances []GarageHostInstance `yaml:"instances"`
 }
@@ -32,12 +37,12 @@ type GarageHost struct {
 // Host consolidates all information about a single host from the bootstrap
 // file.
 type Host struct {
-	Name   string
+	Name   string      `yaml:"name"`
-	Nebula NebulaHost
+	Nebula NebulaHost  `yaml:"nebula"`
-	Garage *GarageHost
+	Garage *GarageHost `yaml:"garage,omitempty"`
 }
 
-func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
+func loadHostsLegacy(bootstrapFS fs.FS) (map[string]Host, error) {
 
 	hosts := map[string]Host{}
 
@@ -51,7 +56,7 @@ func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
 	}
 
 	{
-		globPath := filepath.Join(NebulaHostsDirPath, "*.yml")
+		globPath := "nebula/hosts/*.yml"
 
 		nebulaHostFiles, err := fs.Glob(bootstrapFS, globPath)
 		if err != nil {
@@ -77,7 +82,7 @@ func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
 
 	for hostName, host := range hosts {
 
-		garageHostPath := filepath.Join(GarageHostsDirPath, hostName+".yml")
+		garageHostPath := filepath.Join("garage/hosts", hostName+".yml")
 
 		var garageHost GarageHost
 		if err := readAsYaml(&garageHost, garageHostPath); errors.Is(err, fs.ErrNotExist) {
@@ -92,3 +97,55 @@ func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
 
 	return hosts, nil
 }
+
+func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
+
+	hosts := map[string]Host{}
+
+	readAsYaml := func(into interface{}, path string) error {
+		b, err := fs.ReadFile(bootstrapFS, path)
+		if err != nil {
+			return fmt.Errorf("reading file from fs: %w", err)
+		}
+
+		return yaml.Unmarshal(b, into)
+	}
+
+	globPath := filepath.Join(hostsDirPath, "*.yml")
+
+	hostPaths, err := fs.Glob(bootstrapFS, globPath)
+	if err != nil {
+		return nil, fmt.Errorf("listing host files at %q in fs: %w", globPath, err)
+	}
+
+	for _, hostPath := range hostPaths {
+
+		hostName := filepath.Base(hostPath)
+		hostName = strings.TrimSuffix(hostName, filepath.Ext(hostName))
+
+		var host Host
+		if err := readAsYaml(&host, hostPath); err != nil {
+			return nil, fmt.Errorf("reading %q as yaml: %w", hostPath, err)
+		}
+
+		hosts[hostName] = host
+	}
+
+	if len(hosts) > 0 {
+		return hosts, nil
+	}
+
+	// We used to have the bootstrap file laid out differently. If no hosts were
+	// found then the bootstrap file is probably in that format.
+	hosts, err = loadHostsLegacy(bootstrapFS)
+
+	if err != nil {
+		return nil, fmt.Errorf("loading hosts in legacy layout from fs: %w", err)
+	}
+
+	if len(hosts) == 0 {
+		return nil, fmt.Errorf("failed to load any hosts from fs")
+	}
+
+	return hosts, nil
+}

go-workspace/src/bootstrap/nebula.go

@@ -2,9 +2,7 @@ package bootstrap
 
 // Paths within the bootstrap FS related to nebula.
 const (
-	NebulaHostsDirPath = "nebula/hosts"
+	nebulaCertsCACertPath   = "nebula/certs/ca.crt"
-
+	nebulaCertsHostCertPath = "nebula/certs/host.crt"
-	NebulaCertsCACertPath   = "nebula/certs/ca.crt"
+	nebulaCertsHostKeyPath  = "nebula/certs/host.key"
-	NebulaCertsHostCertPath = "nebula/certs/host.crt"
-	NebulaCertsHostKeyPath  = "nebula/certs/host.key"
 )

go-workspace/src/cmd/cryptic-net-main/main.go

@@ -19,9 +19,8 @@ import (
 	garage_entrypoint "cryptic-net/cmd/garage-entrypoint"
 	garage_layout_diff "cryptic-net/cmd/garage-layout-diff"
 	garage_peer_keygen "cryptic-net/cmd/garage-peer-keygen"
-	garage_update_global_bucket "cryptic-net/cmd/garage-update-global-bucket"
 	nebula_entrypoint "cryptic-net/cmd/nebula-entrypoint"
-	nebula_update_global_bucket "cryptic-net/cmd/nebula-update-global-bucket"
+	update_global_bucket "cryptic-net/cmd/update-global-bucket"
 	"fmt"
 	"os"
 )
@@ -36,9 +35,8 @@ var mainFns = []mainFn{
 	{"garage-entrypoint", garage_entrypoint.Main},
 	{"garage-layout-diff", garage_layout_diff.Main},
 	{"garage-peer-keygen", garage_peer_keygen.Main},
-	{"garage-update-global-bucket", garage_update_global_bucket.Main},
 	{"nebula-entrypoint", nebula_entrypoint.Main},
-	{"nebula-update-global-bucket", nebula_update_global_bucket.Main},
+	{"update-global-bucket", update_global_bucket.Main},
 }
 
 var mainFnsMap = func() map[string]mainFn {

go-workspace/src/cmd/entrypoint/daemon.go

@@ -7,14 +7,16 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"os"
 	"path/filepath"
+	"strconv"
 	"sync"
 	"time"
 
 	crypticnet "cryptic-net"
 	"cryptic-net/bootstrap"
-	bootstrap_creator "cryptic-net/bootstrap/creator"
+	"cryptic-net/garage"
 	"cryptic-net/yamlutil"
 
 	"github.com/cryptic-io/pmux/pmuxlib"
@@ -108,21 +110,30 @@ func writeBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
 // creates a new bootstrap file using available information from the network. If
 // the new bootstrap file is different than the existing one, the existing one
 // is overwritten, ReloadBootstrap is called on env, true is returned.
-func reloadBootstrap(env *crypticnet.Env) (bool, error) {
+func reloadBootstrap(env *crypticnet.Env, s3Client garage.S3APIClient) (bool, error) {
+
+	newHosts, err := bootstrap.GetGarageBootstrapHosts(env.Context, s3Client)
+	if err != nil {
+		return false, fmt.Errorf("getting hosts from garage: %w", err)
+	}
+
+	newHostsHash, err := bootstrap.HostsHash(newHosts)
+	if err != nil {
+		return false, fmt.Errorf("calculating hash of new hosts: %w", err)
+	}
+
+	currHostsHash, err := bootstrap.HostsHash(env.Bootstrap.Hosts)
+	if err != nil {
+		return false, fmt.Errorf("calculating hash of current hosts: %w", err)
+	}
+
+	if bytes.Equal(newHostsHash, currHostsHash) {
+		return false, nil
+	}
 
 	buf := new(bytes.Buffer)
-
+	if err := env.Bootstrap.WithHosts(newHosts).WriteTo(buf); err != nil {
-	if err := bootstrap_creator.NewForThisHost(env, buf); err != nil {
+		return false, fmt.Errorf("writing new bootstrap file: %w", err)
-		return false, fmt.Errorf("generating new bootstrap from env: %w", err)
-	}
-
-	newBootstrap, err := bootstrap.FromReader(bytes.NewReader(buf.Bytes()))
-	if err != nil {
-		return false, fmt.Errorf("parsing bootstrap which was just created: %w", err)
-	}
-
-	if bytes.Equal(newBootstrap.Hash, env.Bootstrap.Hash) {
-		return false, nil
 	}
 
 	if err := writeBootstrapToDataDir(env, buf); err != nil {
@@ -135,9 +146,10 @@ func reloadBootstrap(env *crypticnet.Env) (bool, error) {
 // runs a single pmux process ofor daemon, returning only once the env.Context
 // has been canceled or bootstrap info has been changed. This will always block
 // until the spawned pmux has returned.
-func runDaemonPmuxOnce(env *crypticnet.Env) error {
+func runDaemonPmuxOnce(env *crypticnet.Env, s3Client garage.S3APIClient) error {
 
 	thisHost := env.Bootstrap.ThisHost()
+	thisDaemon := env.ThisDaemon()
 	fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP)
 
 	pmuxProcConfigs := []pmuxlib.ProcessConfig{
@@ -154,13 +166,39 @@ func runDaemonPmuxOnce(env *crypticnet.Env) error {
 			Args: []string{
 				"wait-for-ip",
 				thisHost.Nebula.IP,
-				"bash",
 				"dnsmasq-entrypoint",
 			},
 		},
 	}
 
-	if len(env.ThisDaemon().Storage.Allocations) > 0 {
+	{
+		var args []string
+
+		if allocs := thisDaemon.Storage.Allocations; len(allocs) > 0 {
+			for _, alloc := range allocs {
+				args = append(
+					args,
+					"wait-for",
+					net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
+					"--",
+				)
+			}
+		} else {
+			args = []string{
+				"wait-for-ip",
+				thisHost.Nebula.IP,
+			}
+		}
+
+		pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
+			Name:        "update-global-bucket",
+			Cmd:         "bash",
+			Args:        append(args, "update-global-bucket"),
+			NoRestartOn: []int{0},
+		})
+	}
+
+	if len(thisDaemon.Storage.Allocations) > 0 {
 		pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
 			Name: "garage",
 			Cmd:  "bash",
@@ -204,7 +242,7 @@ func runDaemonPmuxOnce(env *crypticnet.Env) error {
 
 			fmt.Fprintln(os.Stderr, "checking for changes to bootstrap")
 
-			if changed, err := reloadBootstrap(env); err != nil {
+			if changed, err := reloadBootstrap(env, s3Client); err != nil {
 				return fmt.Errorf("reloading bootstrap: %w", err)
 
 			} else if changed {
@@ -243,6 +281,11 @@ var subCmdDaemon = subCmd{
 
 		env := subCmdCtx.env
 
+		s3Client, err := env.GlobalBucketS3APIClient()
+		if err != nil {
+			return fmt.Errorf("creating client for global bucket: %w", err)
+		}
+
 		appDirPath := env.AppDirPath
 
 		builtinDaemonYmlPath := filepath.Join(appDirPath, "etc", "daemon.yml")
@@ -326,7 +369,7 @@ var subCmdDaemon = subCmd{
 
 		for {
 
-			if err := runDaemonPmuxOnce(env); errors.Is(err, context.Canceled) {
+			if err := runDaemonPmuxOnce(env, s3Client); errors.Is(err, context.Canceled) {
 				return nil
 
 			} else if err != nil {

go-workspace/src/cmd/entrypoint/hosts.go

@@ -1,29 +1,20 @@
 package entrypoint
 
 import (
-	"bytes"
 	"cryptic-net/bootstrap"
-	bootstrap_creator "cryptic-net/bootstrap/creator"
+	"cryptic-net/nebula"
-	"cryptic-net/garage"
 	"cryptic-net/tarutil"
 	"errors"
 	"fmt"
 	"io/fs"
 	"net"
 	"os"
-	"path/filepath"
 	"regexp"
+	"sort"
 
-	"github.com/minio/minio-go/v7"
 	"gopkg.in/yaml.v3"
 )
 
-const nebulaHostPathPrefix = "nebula/hosts/"
-
-func nebulaHostPath(name string) string {
-	return filepath.Join(nebulaHostPathPrefix, name+".yml")
-}
-
 var hostNameRegexp = regexp.MustCompile(`^[a-z][a-z0-9\-]*$`)
 
 func validateHostName(name string) error {
@@ -78,30 +69,14 @@ var subCmdHostsAdd = subCmd{
 			return fmt.Errorf("creating client for global bucket: %w", err)
 		}
 
-		nebulaHost := bootstrap.NebulaHost{
+		host := bootstrap.Host{
 			Name: *name,
+			Nebula: bootstrap.NebulaHost{
 				IP: *ip,
+			},
 		}
 
-		bodyBuf := new(bytes.Buffer)
+		return bootstrap.PutGarageBoostrapHost(env.Context, client, host)
-
-		if err := yaml.NewEncoder(bodyBuf).Encode(nebulaHost); err != nil {
-			return fmt.Errorf("marshaling nebula host to yaml: %w", err)
-		}
-
-		filePath := nebulaHostPath(*name)
-
-		_, err = client.PutObject(
-			env.Context, garage.GlobalBucket, filePath,
-			bodyBuf, int64(bodyBuf.Len()),
-			minio.PutObjectOptions{},
-		)
-
-		if err != nil {
-			return fmt.Errorf("writing to %q in global bucket: %w", filePath, err)
-		}
-
-		return nil
 	},
 }
 
@@ -118,51 +93,19 @@ var subCmdHostsList = subCmd{
 			return fmt.Errorf("creating client for global bucket: %w", err)
 		}
 
-		objInfoCh := client.ListObjects(
+		hostsMap, err := bootstrap.GetGarageBootstrapHosts(env.Context, client)
-			env.Context, garage.GlobalBucket,
-			minio.ListObjectsOptions{
-				Prefix: nebulaHostPathPrefix,
-			},
-		)
-
-		for {
-			select {
-			case <-env.Context.Done():
-				return env.Context.Err()
-
-			case objInfo, ok := <-objInfoCh:
-
-				if !ok {
-					return nil
-				} else if objInfo.Err != nil {
-					return objInfo.Err
-				}
-
-				obj, err := client.GetObject(
-					env.Context, garage.GlobalBucket, objInfo.Key,
-					minio.GetObjectOptions{},
-				)
-
 		if err != nil {
-					return fmt.Errorf("retrieving object %q from global bucket: %w", objInfo.Key, err)
+			return fmt.Errorf("retrieving hosts from garage: %w", err)
 		}
 
-				var nebulaHost bootstrap.NebulaHost
+		hosts := make([]bootstrap.Host, 0, len(hostsMap))
-
+		for _, host := range hostsMap {
-				err = yaml.NewDecoder(obj).Decode(&nebulaHost)
+			hosts = append(hosts, host)
-				obj.Close()
-
-				if err != nil {
-					return fmt.Errorf("yaml decoding %q from global bucket: %w", objInfo.Key, err)
 		}
 
-				fmt.Fprintf(
+		sort.Slice(hosts, func(i, j int) bool { return hosts[i].Name < hosts[j].Name })
-					os.Stdout, "%s\t%s\n",
-					nebulaHost.Name, nebulaHost.IP,
-				)
-			}
-		}
 
+		return yaml.NewEncoder(os.Stdout).Encode(hosts)
 	},
 }
 
@@ -176,7 +119,7 @@ var subCmdHostsDelete = subCmd{
 
 		name := flags.StringP(
 			"name", "n", "",
-			"Name of the new host",
+			"Name of the host to delete",
 		)
 
 		if err := flags.Parse(subCmdCtx.args); err != nil {
@@ -189,25 +132,12 @@ var subCmdHostsDelete = subCmd{
 
 		env := subCmdCtx.env
 
-		filePath := nebulaHostPath(*name)
-
 		client, err := env.GlobalBucketS3APIClient()
 		if err != nil {
 			return fmt.Errorf("creating client for global bucket: %w", err)
 		}
 
-		err = client.RemoveObject(
+		return bootstrap.RemoveGarageBootstrapHost(env.Context, client, *name)
-			env.Context, garage.GlobalBucket, filePath,
-			minio.RemoveObjectOptions{},
-		)
-
-		if garage.IsKeyNotFound(err) {
-			return fmt.Errorf("host %q not found", *name)
-		} else if err != nil {
-			return fmt.Errorf("removing object %q from global bucket: %w", filePath, err)
-		}
-
-		return nil
 	},
 }
 
@@ -258,12 +188,51 @@ var subCmdHostsMakeBootstrap = subCmd{
 			return errors.New("--name and --admin-path are required")
 		}
 
+		env := subCmdCtx.env
+
 		adminFS, err := readAdminFS(*adminPath)
 		if err != nil {
 			return fmt.Errorf("reading admin.tgz with --admin-path of %q: %w", *adminPath, err)
 		}
 
-		return bootstrap_creator.NewForHost(subCmdCtx.env, adminFS, *name, os.Stdout)
+		client, err := env.GlobalBucketS3APIClient()
+		if err != nil {
+			return fmt.Errorf("creating client for global bucket: %w", err)
+		}
+
+		// NOTE this isn't _technically_ required, but if the `hosts add`
+		// command for this host has been run recently then it might not have
+		// made it into the bootstrap file yet, and so won't be in
+		// `env.Bootstrap`.
+		hosts, err := bootstrap.GetGarageBootstrapHosts(env.Context, client)
+		if err != nil {
+			return fmt.Errorf("retrieving host info from garage: %w", err)
+		}
+
+		host, ok := hosts[*name]
+		if !ok {
+			return fmt.Errorf("couldn't find host into for %q in garage, has `cryptic-net hosts add` been run yet?", *name)
+		}
+
+		nebulaHostCert, err := nebula.NewHostCert(adminFS, host)
+		if err != nil {
+			return fmt.Errorf("creating new nebula host key/cert: %w", err)
+		}
+
+		newBootstrap := bootstrap.Bootstrap{
+			Hosts:    hosts,
+			HostName: *name,
+
+			NebulaCertsCACert:   nebulaHostCert.CACert,
+			NebulaCertsHostCert: nebulaHostCert.HostCert,
+			NebulaCertsHostKey:  nebulaHostCert.HostKey,
+
+			// TODO these should use adminFS
+			GarageRPCSecret:                    env.Bootstrap.GarageRPCSecret,
+			GarageGlobalBucketS3APICredentials: env.Bootstrap.GarageGlobalBucketS3APICredentials,
+		}
+
+		return newBootstrap.WriteTo(os.Stdout)
 	},
 }
 

go-workspace/src/cmd/garage-entrypoint/main.go

@@ -125,12 +125,5 @@ func Main() {
 		NoRestartOn: []int{0},
 	})
 
-	pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
-		Name:        "garage-update-global-bucket",
-		Cmd:         "bash",
-		Args:        waitForArgs(env, "cryptic-net-main", "garage-update-global-bucket"),
-		NoRestartOn: []int{0},
-	})
-
 	pmuxlib.Run(env.Context, pmuxlib.Config{Processes: pmuxProcConfigs})
 }

go-workspace/src/cmd/garage-update-global-bucket/main.go

@@ -1,90 +0,0 @@
-package garage_update_global_bucket
-
-import (
-	"bytes"
-	crypticnet "cryptic-net"
-	"cryptic-net/bootstrap"
-	"cryptic-net/garage"
-	"fmt"
-	"log"
-	"path/filepath"
-
-	"github.com/minio/minio-go/v7"
-	"gopkg.in/yaml.v3"
-)
-
-func updateGlobalBucket(env *crypticnet.Env) error {
-
-	ctx := env.Context
-
-	client, err := env.GlobalBucketS3APIClient()
-	if err != nil {
-		return fmt.Errorf("creating client for global bucket: %w", err)
-	}
-
-	filePath := filepath.Join(
-		"garage/hosts",
-		env.Bootstrap.HostName+".yml",
-	)
-
-	daemon := env.ThisDaemon()
-
-	if len(daemon.Storage.Allocations) == 0 {
-
-		err := client.RemoveObject(
-			ctx, garage.GlobalBucket, filePath,
-			minio.RemoveObjectOptions{},
-		)
-
-		if garage.IsKeyNotFound(err) {
-			return nil
-		} else if err != nil {
-			return fmt.Errorf("removing %q from global bucket: %w", filePath, err)
-		}
-
-		return nil
-	}
-
-	var garageHost bootstrap.GarageHost
-
-	for _, alloc := range daemon.Storage.Allocations {
-
-		garageHostInstance := bootstrap.GarageHostInstance{
-			RPCPort:   alloc.RPCPort,
-			S3APIPort: alloc.S3APIPort,
-			WebPort:   alloc.WebPort,
-		}
-
-		garageHost.Instances = append(garageHost.Instances, garageHostInstance)
-	}
-
-	buf := new(bytes.Buffer)
-
-	if err := yaml.NewEncoder(buf).Encode(garageHost); err != nil {
-		return fmt.Errorf("yaml encoding garage host data: %w", err)
-	}
-
-	_, err = client.PutObject(
-		ctx, garage.GlobalBucket, filePath, buf, int64(buf.Len()),
-		minio.PutObjectOptions{},
-	)
-
-	if err != nil {
-		return fmt.Errorf("writing to %q in global bucket: %w", filePath, err)
-	}
-
-	return nil
-}
-
-func Main() {
-
-	env, err := crypticnet.ReadEnv()
-
-	if err != nil {
-		log.Fatalf("reading envvars: %v", err)
-	}
-
-	if err := updateGlobalBucket(env); err != nil {
-		log.Fatalf("updating global bucket: %v", err)
-	}
-}

go-workspace/src/cmd/nebula-entrypoint/main.go

@@ -4,12 +4,12 @@ import (
 	"cryptic-net/yamlutil"
 	"log"
 	"net"
+	"os"
 	"path/filepath"
 	"strconv"
+	"syscall"
 
 	crypticnet "cryptic-net"
-
-	"github.com/cryptic-io/pmux/pmuxlib"
 )
 
 func Main() {
@@ -122,19 +122,13 @@ func Main() {
 		log.Fatalf("writing nebula.yml to %q: %v", nebulaYmlPath, err)
 	}
 
-	pmuxlib.Run(env.Context, pmuxlib.Config{Processes: []pmuxlib.ProcessConfig{
+	var (
-		{
+		binPath = env.BinPath("nebula")
-			Name: "nebula-update-global-bucket",
+		args    = []string{"nebula", "-config", nebulaYmlPath}
-			Cmd:  "cryptic-net-main",
+		cliEnv  = os.Environ()
-			Args: []string{
+	)
-				"nebula-update-global-bucket",
+
-			},
+	if err := syscall.Exec(binPath, args, cliEnv); err != nil {
-			NoRestartOn: []int{0},
+		log.Fatalf("calling exec(%q, %#v, %#v)", binPath, args, cliEnv)
-		},
+	}
-		{
-			Name: "nebula",
-			Cmd:  "nebula",
-			Args: []string{"-config", nebulaYmlPath},
-		},
-	}})
 }

go-workspace/src/cmd/nebula-update-global-bucket/main.go

@@ -1,62 +0,0 @@
-package nebula_update_global_bucket
-
-import (
-	"bytes"
-	crypticnet "cryptic-net"
-	"cryptic-net/garage"
-	"fmt"
-	"log"
-	"path/filepath"
-
-	"github.com/minio/minio-go/v7"
-	"gopkg.in/yaml.v3"
-)
-
-func updateGlobalBucket(env *crypticnet.Env) error {
-
-	ctx := env.Context
-
-	client, err := env.GlobalBucketS3APIClient()
-	if err != nil {
-		return fmt.Errorf("creating client for global bucket: %w", err)
-	}
-
-	daemon := env.ThisDaemon()
-
-	host := env.Bootstrap.ThisHost()
-
-	host.Nebula.Name = host.Name
-	host.Nebula.PublicAddr = daemon.VPN.PublicAddr
-
-	buf := new(bytes.Buffer)
-
-	if err := yaml.NewEncoder(buf).Encode(host.Nebula); err != nil {
-		return fmt.Errorf("yaml encoding garage host data: %w", err)
-	}
-
-	filePath := filepath.Join("nebula/hosts", host.Name+".yml")
-
-	_, err = client.PutObject(
-		ctx, garage.GlobalBucket, filePath, buf, int64(buf.Len()),
-		minio.PutObjectOptions{},
-	)
-
-	if err != nil {
-		return fmt.Errorf("writing to %q in global bucket: %w", filePath, err)
-	}
-
-	return nil
-}
-
-func Main() {
-
-	env, err := crypticnet.ReadEnv()
-
-	if err != nil {
-		log.Fatalf("reading envvars: %v", err)
-	}
-
-	if err := updateGlobalBucket(env); err != nil {
-		log.Fatalf("updating global bucket: %v", err)
-	}
-}

go-workspace/src/cmd/update-global-bucket/main.go

@@ -0,0 +1,54 @@
+package update_global_bucket
+
+import (
+	crypticnet "cryptic-net"
+	"cryptic-net/bootstrap"
+	"log"
+)
+
+func Main() {
+
+	env, err := crypticnet.ReadEnv()
+	if err != nil {
+		log.Fatalf("reading envvars: %v", err)
+	}
+
+	client, err := env.GlobalBucketS3APIClient()
+	if err != nil {
+		log.Fatalf("creating client for global bucket: %v", err)
+	}
+
+	host := env.Bootstrap.ThisHost()
+
+	// We update the Host for this host in place, prior to writing it via the
+	// bootstrap method. We want to ensure that any changes made via daemon are
+	// reflected into the bootstrap data which is pushed up.
+	//
+	// TODO it'd be better if this was done within the daemon command itself,
+	// prior to any sub-processes being started. This would help us avoid this
+	// weird logic here, and would prevent all sub-processes from needing to be
+	// restarted the first time the daemon is started after daemon.yml is
+	// modified.
+	daemon := env.ThisDaemon()
+
+	host.Nebula.PublicAddr = daemon.VPN.PublicAddr
+
+	host.Garage = nil
+
+	if allocs := daemon.Storage.Allocations; len(allocs) > 0 {
+
+		host.Garage = new(bootstrap.GarageHost)
+
+		for _, alloc := range allocs {
+			host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
+				RPCPort:   alloc.RPCPort,
+				S3APIPort: alloc.S3APIPort,
+				WebPort:   alloc.WebPort,
+			})
+		}
+	}
+
+	if err := bootstrap.PutGarageBoostrapHost(env.Context, client, host); err != nil {
+		log.Fatal(err)
+	}
+}

go-workspace/src/nebula/nebula.go

@@ -28,22 +28,22 @@ var ipCIDRMask = func() net.IPMask {
 // HostCert contains the certificate and private key files which will need to
 // be present on a particular host. Each file is PEM encoded.
 type HostCert struct {
-	CACert   []byte
+	CACert   string
-	HostKey  []byte
+	HostKey  string
-	HostCert []byte
+	HostCert string
 }
 
 // CACert contains the certificate and private files which can be used to create
 // HostCerts. Each file is PEM encoded.
 type CACert struct {
-	CACert []byte
+	CACert string
-	CAKey  []byte
+	CAKey  string
 }
 
 // NewHostCert generates a new key/cert for a nebula host using the CA key
 // which will be found in the adminFS.
 func NewHostCert(
-	adminFS fs.FS, host bootstrap.NebulaHost,
+	adminFS fs.FS, host bootstrap.Host,
 ) (
 	HostCert, error,
 ) {
@@ -78,9 +78,9 @@ func NewHostCert(
 
 	expireAt := caCrt.Details.NotAfter.Add(-1 * time.Second)
 
-	ip := net.ParseIP(host.IP)
+	ip := net.ParseIP(host.Nebula.IP)
 	if ip == nil {
-		return HostCert{}, fmt.Errorf("invalid host ip %q", host.IP)
+		return HostCert{}, fmt.Errorf("invalid host ip %q", host.Nebula.IP)
 	}
 
 	ipNet := &net.IPNet{
@@ -126,9 +126,9 @@ func NewHostCert(
 	}
 
 	return HostCert{
-		CACert:   caCrtPEM,
+		CACert:   string(caCrtPEM),
-		HostKey:  hostKeyPEM,
+		HostKey:  string(hostKeyPEM),
-		HostCert: hostCrtPEM,
+		HostCert: string(hostCrtPEM),
 	}, nil
 }
 
@@ -166,7 +166,7 @@ func NewCACert(domain string) (CACert, error) {
 	}
 
 	return CACert{
-		CACert: caCrtPem,
+		CACert: string(caCrtPem),
-		CAKey:  caKeyPEM,
+		CAKey:  string(caKeyPEM),
 	}, nil
 }