Refactor how bootstrap files are created

The new code makes it a lot clearer what the sources of each
file/directory is, and makes it more difficult to forget to add a file
or directory. This will be helpful when it comes to bootstrapping an
entire network, which will require yet a third way of generating
bootstrap files.

go-workspace/src/bootstrap/bootstrap.go

@@ -2,6 +2,10 @@
 package bootstrap
 
 import (
+	"context"
+	crypticnet "cryptic-net"
+	"cryptic-net/garage"
+	"cryptic-net/nebula"
 	"cryptic-net/tarutil"
 	"fmt"
 	"io"
@@ -32,3 +36,96 @@ func GetHashFromReader(r io.Reader) ([]byte, error) {
 
 	return GetHashFromFS(bootstrapFS)
 }
+
+func newBootstrap(
+	ctx context.Context,
+	into io.Writer,
+	hostname provider,
+	nebulaCerts provider,
+	nebulaHosts provider,
+	garageRPCSecret provider,
+	garageGlobalBucketKey provider,
+	garageHosts provider,
+) error {
+
+	pairs := []struct {
+		path     string
+		provider provider
+	}{
+		{"hostname", hostname},
+		{"nebula/certs", nebulaCerts},
+		{"nebula/hosts", nebulaHosts},
+		{"garage/rpc-secret.txt", garageRPCSecret},
+		{"garage/global-bucket-key.yml", garageGlobalBucketKey},
+		{"garage/hosts", garageHosts},
+	}
+
+	w := tarutil.NewTGZWriter(into)
+
+	for _, pair := range pairs {
+
+		if err := pair.provider(ctx, w, pair.path); err != nil {
+			return fmt.Errorf("populating %q in new bootstrap: %w", pair.path, err)
+		}
+	}
+
+	return w.Close()
+}
+
+// NewForThisHost generates a new bootstrap file for the current host, based on
+// the existing environment as well as data in garage.
+func NewForThisHost(env *crypticnet.Env, into io.Writer) error {
+
+	client, err := garage.GlobalBucketAPIClient(env)
+	if err != nil {
+		return fmt.Errorf("creating client for global bucket: %w", err)
+	}
+
+	return newBootstrap(
+		env.Context,
+		into,
+		provideFromFS(env.BootstrapFS),    // hostname
+		provideDirFromFS(env.BootstrapFS), // nebulaCerts
+		provideDirFromGarage(client),      // nebulaHosts
+		provideFromFS(env.BootstrapFS),    // garageRPCSecret
+		provideFromFS(env.BootstrapFS),    // garageGlobalBucketKey
+		provideDirFromGarage(client),      // garageHosts
+	)
+}
+
+// NewForHost generates a new bootstrap file for an arbitrary host, based on the
+// given admin file's FS and data in garage.
+func NewForHost(env *crypticnet.Env, adminFS fs.FS, name string, into io.Writer) error {
+
+	host, ok := env.Hosts[name]
+	if !ok {
+		return fmt.Errorf("unknown host %q, make sure host entry has been created", name)
+	}
+
+	client, err := garage.GlobalBucketAPIClient(env)
+	if err != nil {
+		return fmt.Errorf("creating client for global bucket: %w", err)
+	}
+
+	nebulaHostCert, err := nebula.NewHostCert(adminFS, host.Nebula)
+	if err != nil {
+		return fmt.Errorf("creating new nebula host key/cert: %w", err)
+	}
+
+	nebulaHostCertDir := map[string][]byte{
+		"ca.crt":   nebulaHostCert.CACert,
+		"host.key": nebulaHostCert.HostKey,
+		"host.crt": nebulaHostCert.HostCert,
+	}
+
+	return newBootstrap(
+		env.Context,
+		into,
+		provideFromBytes([]byte(name)),       // hostname
+		provideDirFromMap(nebulaHostCertDir), // nebulaCerts
+		provideDirFromGarage(client),         // nebulaHosts
+		provideFromFS(adminFS),               // garageRPCSecret
+		provideFromFS(adminFS),               // garageGlobalBucketKey
+		provideDirFromGarage(client),         // garageHosts
+	)
+}

go-workspace/src/bootstrap/new_for_host.go

@@ -1,161 +0,0 @@
-package bootstrap
-
-import (
-	crypticnet "cryptic-net"
-	"cryptic-net/garage"
-	"cryptic-net/tarutil"
-	"crypto/rand"
-	"fmt"
-	"io"
-	"io/fs"
-	"net"
-	"time"
-
-	"github.com/slackhq/nebula/cert"
-	"golang.org/x/crypto/curve25519"
-)
-
-var ipCIDRMask = func() net.IPMask {
-	_, ipNet, err := net.ParseCIDR("10.10.0.0/16")
-	if err != nil {
-		panic(err)
-	}
-	return ipNet.Mask
-}()
-
-// Generates a new key/cert for a nebula host, writing their encoded forms into
-// the given TGZWriter. It will also write the ca.crt file to the TGZWriter.
-//
-// The logic here is largely based on
-// https://github.com/slackhq/nebula/blob/v1.4.0/cmd/nebula-cert/sign.go
-func writeNewNebulaCert(
-	w *tarutil.TGZWriter, adminFS fs.FS, host crypticnet.NebulaHost,
-) error {
-
-	caKeyPEM, err := fs.ReadFile(adminFS, "nebula/certs/ca.key")
-	if err != nil {
-		return fmt.Errorf("reading ca.key from admin fs: %w", err)
-	}
-
-	caKey, _, err := cert.UnmarshalEd25519PrivateKey(caKeyPEM)
-	if err != nil {
-		return fmt.Errorf("unmarshaling ca.key: %w", err)
-	}
-
-	caCrtPEM, err := fs.ReadFile(adminFS, "nebula/certs/ca.crt")
-	if err != nil {
-		return fmt.Errorf("reading ca.crt from admin fs: %w", err)
-	}
-
-	caCrt, _, err := cert.UnmarshalNebulaCertificateFromPEM(caCrtPEM)
-	if err != nil {
-		return fmt.Errorf("unmarshaling ca.crt: %w", err)
-	}
-
-	issuer, err := caCrt.Sha256Sum()
-	if err != nil {
-		return fmt.Errorf("getting ca.crt issuer: %w", err)
-	}
-
-	expireAt := caCrt.Details.NotAfter.Add(-1 * time.Second)
-
-	ip := net.ParseIP(host.IP)
-	if ip == nil {
-		return fmt.Errorf("invalid host ip %q", host.IP)
-	}
-
-	ipNet := &net.IPNet{
-		IP:   ip,
-		Mask: ipCIDRMask,
-	}
-
-	var hostPub, hostKey []byte
-	{
-		var pubkey, privkey [32]byte
-		if _, err := io.ReadFull(rand.Reader, privkey[:]); err != nil {
-			return fmt.Errorf("reading random bytes to form private key: %w", err)
-		}
-		curve25519.ScalarBaseMult(&pubkey, &privkey)
-		hostPub, hostKey = pubkey[:], privkey[:]
-	}
-
-	hostCrt := cert.NebulaCertificate{
-		Details: cert.NebulaCertificateDetails{
-			Name:      host.Name,
-			Ips:       []*net.IPNet{ipNet},
-			NotBefore: time.Now(),
-			NotAfter:  expireAt,
-			PublicKey: hostPub,
-			IsCA:      false,
-			Issuer:    issuer,
-		},
-	}
-
-	if err := hostCrt.CheckRootConstrains(caCrt); err != nil {
-		return fmt.Errorf("validating certificate constraints: %w", err)
-	}
-
-	if err := hostCrt.Sign(caKey); err != nil {
-		return fmt.Errorf("signing host cert with ca.key: %w", err)
-	}
-
-	hostKeyPEM := cert.MarshalX25519PrivateKey(hostKey)
-
-	hostCrtPEM, err := hostCrt.MarshalToPEM()
-	if err != nil {
-		return fmt.Errorf("marshalling host.crt: %w", err)
-	}
-
-	w.WriteFileBytes("nebula/certs/ca.crt", caCrtPEM)
-	w.WriteFileBytes("nebula/certs/host.key", hostKeyPEM)
-	w.WriteFileBytes("nebula/certs/host.crt", hostCrtPEM)
-
-	return nil
-}
-
-// NewForHost generates a new bootstrap file for an arbitrary host, based on the
-// given admin file's FS and data in garage.
-func NewForHost(env *crypticnet.Env, adminFS fs.FS, name string, into io.Writer) error {
-
-	host, ok := env.Hosts[name]
-	if !ok {
-		return fmt.Errorf("unknown host %q, make sure host entry has been created", name)
-	}
-
-	client, err := garage.GlobalBucketAPIClient(env)
-	if err != nil {
-		return fmt.Errorf("creating client for global bucket: %w", err)
-	}
-
-	w := tarutil.NewTGZWriter(into)
-
-	w.WriteFileBytes("hostname", []byte(name))
-
-	if err := writeNewNebulaCert(w, adminFS, host.Nebula); err != nil {
-		return fmt.Errorf("creating/adding host's nebula certs: %w", err)
-	}
-
-	fsFilesToCopy := []string{
-		"garage/rpc-secret.txt",
-		"garage/cryptic-net-global-bucket-key.yml",
-	}
-
-	for _, filePath := range fsFilesToCopy {
-		if err := copyFSFile(w, adminFS, filePath); err != nil {
-			return fmt.Errorf("copying %q from bootstrap fs: %w", filePath, err)
-		}
-	}
-
-	garageDirsToCopy := []string{
-		"nebula/hosts",
-		"garage/hosts",
-	}
-
-	for _, dirPath := range garageDirsToCopy {
-		if err := copyGarageDir(env.Context, client, w, dirPath); err != nil {
-			return fmt.Errorf("copying %q from garage: %w", dirPath, err)
-		}
-	}
-
-	return w.Close()
-}

go-workspace/src/bootstrap/new_for_this_host.go

@@ -1,49 +0,0 @@
-package bootstrap
-
-import (
-	crypticnet "cryptic-net"
-	"cryptic-net/garage"
-	"cryptic-net/tarutil"
-	"fmt"
-	"io"
-)
-
-// NewForThisHost generates a new bootstrap file for the current host, based on
-// the existing environment as well as data in garage.
-func NewForThisHost(env *crypticnet.Env, into io.Writer) error {
-
-	client, err := garage.GlobalBucketAPIClient(env)
-	if err != nil {
-		return fmt.Errorf("creating client for global bucket: %w", err)
-	}
-
-	w := tarutil.NewTGZWriter(into)
-
-	fsFilesToCopy := []string{
-		"hostname",
-		"nebula/certs/ca.crt",
-		"nebula/certs/host.crt",
-		"nebula/certs/host.key",
-		"garage/rpc-secret.txt",
-		"garage/cryptic-net-global-bucket-key.yml",
-	}
-
-	for _, filePath := range fsFilesToCopy {
-		if err := copyFSFile(w, env.BootstrapFS, filePath); err != nil {
-			return fmt.Errorf("copying %q from bootstrap fs: %w", filePath, err)
-		}
-	}
-
-	garageDirsToCopy := []string{
-		"nebula/hosts",
-		"garage/hosts",
-	}
-
-	for _, dirPath := range garageDirsToCopy {
-		if err := copyGarageDir(env.Context, client, w, dirPath); err != nil {
-			return fmt.Errorf("copying %q from garage: %w", dirPath, err)
-		}
-	}
-
-	return w.Close()
-}

go-workspace/src/bootstrap/provider.go

@@ -0,0 +1,139 @@
+package bootstrap
+
+import (
+	"context"
+	"cryptic-net/garage"
+	"cryptic-net/tarutil"
+	"fmt"
+	"io/fs"
+	"path/filepath"
+
+	"github.com/minio/minio-go/v7"
+)
+
+// provider is a function which will populate the given filePath into the given
+// TGZWriter. The path may be a file or a directory.
+type provider func(context.Context, *tarutil.TGZWriter, string) error
+
+func provideFromBytes(body []byte) provider {
+
+	return func(
+		ctx context.Context,
+		w *tarutil.TGZWriter,
+		filePath string,
+	) error {
+
+		w.WriteFileBytes(filePath, body)
+		return nil
+	}
+}
+
+func provideFromFS(srcFS fs.FS) provider {
+
+	return func(
+		ctx context.Context,
+		w *tarutil.TGZWriter,
+		filePath string,
+	) error {
+
+		return w.CopyFileFromFS(filePath, srcFS)
+	}
+}
+
+func provideDirFromFS(srcFS fs.FS) provider {
+
+	return func(
+		ctx context.Context,
+		w *tarutil.TGZWriter,
+		dirPath string,
+	) error {
+
+		return fs.WalkDir(
+			srcFS, dirPath,
+			func(filePath string, dirEntry fs.DirEntry, err error) error {
+
+				if err != nil {
+					return err
+
+				} else if dirEntry.IsDir() {
+					return nil
+
+				} else if err := w.CopyFileFromFS(filePath, srcFS); err != nil {
+					return fmt.Errorf("copying file %q: %w", filePath, err)
+				}
+
+				return nil
+			},
+		)
+
+	}
+}
+
+func provideDirFromMap(m map[string][]byte) provider {
+
+	return func(
+		ctx context.Context,
+		w *tarutil.TGZWriter,
+		dirPath string,
+	) error {
+
+		for filePath, body := range m {
+			filePath := filepath.Join(dirPath, filePath)
+			w.WriteFileBytes(filePath, body)
+		}
+
+		return nil
+	}
+}
+
+// TODO it'd be great if we could wrap a minio.Client into an fs.FS. That would
+// get rid of a weird dependency in this package, and clean up this code a ton.
+func provideDirFromGarage(client *minio.Client) provider {
+
+	return func(
+		ctx context.Context,
+		w *tarutil.TGZWriter,
+		dirPath string,
+	) error {
+
+		objInfoCh := client.ListObjects(
+			ctx, garage.GlobalBucket,
+			minio.ListObjectsOptions{
+				Prefix:    dirPath,
+				Recursive: true,
+			},
+		)
+
+		for objInfo := range objInfoCh {
+
+			if objInfo.Err != nil {
+				return fmt.Errorf("listing objects: %w", objInfo.Err)
+			}
+
+			obj, err := client.GetObject(
+				ctx, garage.GlobalBucket, objInfo.Key, minio.GetObjectOptions{},
+			)
+
+			if err != nil {
+				return fmt.Errorf(
+					"retrieving object %q from global bucket: %w",
+					objInfo.Key, err,
+				)
+			}
+
+			objStat, err := obj.Stat()
+			if err != nil {
+				obj.Close()
+				return fmt.Errorf(
+					"stating object %q from global bucket: %w",
+					objInfo.Key, err,
+				)
+			}
+
+			w.WriteFile(objInfo.Key, objStat.Size, obj)
+			obj.Close()
+		}
+
+		return nil
+	}
+}

go-workspace/src/bootstrap/util.go

@@ -1,74 +0,0 @@
-package bootstrap
-
-import (
-	"context"
-	"cryptic-net/garage"
-	"cryptic-net/tarutil"
-	"fmt"
-	"io/fs"
-
-	"github.com/minio/minio-go/v7"
-)
-
-func copyFSFile(w *tarutil.TGZWriter, srcFS fs.FS, path string) error {
-
-	f, err := srcFS.Open(path)
-	if err != nil {
-		return fmt.Errorf("opening %q in bootstrap fs: %w", path, err)
-	}
-	defer f.Close()
-
-	fStat, err := f.Stat()
-	if err != nil {
-		return fmt.Errorf("stating %q from bootstrap fs: %w", path, err)
-	}
-
-	w.WriteFile(path, fStat.Size(), f)
-	return nil
-}
-
-func copyGarageDir(
-	ctx context.Context, client *minio.Client,
-	w *tarutil.TGZWriter, path string,
-) error {
-
-	objInfoCh := client.ListObjects(
-		ctx, garage.GlobalBucket,
-		minio.ListObjectsOptions{
-			Prefix:    path,
-			Recursive: true,
-		},
-	)
-
-	for objInfo := range objInfoCh {
-
-		if objInfo.Err != nil {
-			return fmt.Errorf("listing objects: %w", objInfo.Err)
-		}
-
-		obj, err := client.GetObject(
-			ctx, garage.GlobalBucket, objInfo.Key, minio.GetObjectOptions{},
-		)
-
-		if err != nil {
-			return fmt.Errorf(
-				"retrieving object %q from global bucket: %w",
-				objInfo.Key, err,
-			)
-		}
-
-		objStat, err := obj.Stat()
-		if err != nil {
-			obj.Close()
-			return fmt.Errorf(
-				"stating object %q from global bucket: %w",
-				objInfo.Key, err,
-			)
-		}
-
-		w.WriteFile(objInfo.Key, objStat.Size, obj)
-		obj.Close()
-	}
-
-	return nil
-}

go-workspace/src/garage/client.go

@@ -29,7 +29,7 @@ type APICredentials struct {
 // GlobalBucketAPICredentials returns APICredentials for the global bucket.
 func GlobalBucketAPICredentials(env *crypticnet.Env) (APICredentials, error) {
 
-	const path = "garage/cryptic-net-global-bucket-key.yml"
+	const path = "garage/priv/global-bucket-key.yml"
 
 	var creds APICredentials
 	if err := yamlutil.LoadYamlFSFile(&creds, env.BootstrapFS, path); err != nil {

go-workspace/src/nebula/nebula.go

@@ -0,0 +1,125 @@
+// Package nebula contains helper functions and types which are useful for
+// setting up nebula configs, processes, and deployments.
+package nebula
+
+import (
+	crypticnet "cryptic-net"
+	"crypto/rand"
+	"fmt"
+	"io"
+	"io/fs"
+	"net"
+	"time"
+
+	"github.com/slackhq/nebula/cert"
+	"golang.org/x/crypto/curve25519"
+)
+
+// TODO this should one day not be hardcoded
+var ipCIDRMask = func() net.IPMask {
+	_, ipNet, err := net.ParseCIDR("10.10.0.0/16")
+	if err != nil {
+		panic(err)
+	}
+	return ipNet.Mask
+}()
+
+// HostCert contains the certificate and private key files which will need to
+// be present on a particular host. Each file is PEM encoded.
+type HostCert struct {
+	CACert   []byte
+	HostKey  []byte
+	HostCert []byte
+}
+
+// NewHostCert generates a new key/cert for a nebula host using the CA key
+// which will be found in the adminFS.
+func NewHostCert(
+	adminFS fs.FS, host crypticnet.NebulaHost,
+) (
+	HostCert, error,
+) {
+
+	// The logic here is largely based on
+	// https://github.com/slackhq/nebula/blob/v1.4.0/cmd/nebula-cert/sign.go
+
+	caKeyPEM, err := fs.ReadFile(adminFS, "nebula/certs/ca.key")
+	if err != nil {
+		return HostCert{}, fmt.Errorf("reading ca.key from admin fs: %w", err)
+	}
+
+	caKey, _, err := cert.UnmarshalEd25519PrivateKey(caKeyPEM)
+	if err != nil {
+		return HostCert{}, fmt.Errorf("unmarshaling ca.key: %w", err)
+	}
+
+	caCrtPEM, err := fs.ReadFile(adminFS, "nebula/certs/ca.crt")
+	if err != nil {
+		return HostCert{}, fmt.Errorf("reading ca.crt from admin fs: %w", err)
+	}
+
+	caCrt, _, err := cert.UnmarshalNebulaCertificateFromPEM(caCrtPEM)
+	if err != nil {
+		return HostCert{}, fmt.Errorf("unmarshaling ca.crt: %w", err)
+	}
+
+	issuer, err := caCrt.Sha256Sum()
+	if err != nil {
+		return HostCert{}, fmt.Errorf("getting ca.crt issuer: %w", err)
+	}
+
+	expireAt := caCrt.Details.NotAfter.Add(-1 * time.Second)
+
+	ip := net.ParseIP(host.IP)
+	if ip == nil {
+		return HostCert{}, fmt.Errorf("invalid host ip %q", host.IP)
+	}
+
+	ipNet := &net.IPNet{
+		IP:   ip,
+		Mask: ipCIDRMask,
+	}
+
+	var hostPub, hostKey []byte
+	{
+		var pubkey, privkey [32]byte
+		if _, err := io.ReadFull(rand.Reader, privkey[:]); err != nil {
+			return HostCert{}, fmt.Errorf("reading random bytes to form private key: %w", err)
+		}
+		curve25519.ScalarBaseMult(&pubkey, &privkey)
+		hostPub, hostKey = pubkey[:], privkey[:]
+	}
+
+	hostCrt := cert.NebulaCertificate{
+		Details: cert.NebulaCertificateDetails{
+			Name:      host.Name,
+			Ips:       []*net.IPNet{ipNet},
+			NotBefore: time.Now(),
+			NotAfter:  expireAt,
+			PublicKey: hostPub,
+			IsCA:      false,
+			Issuer:    issuer,
+		},
+	}
+
+	if err := hostCrt.CheckRootConstrains(caCrt); err != nil {
+		return HostCert{}, fmt.Errorf("validating certificate constraints: %w", err)
+	}
+
+	if err := hostCrt.Sign(caKey); err != nil {
+		return HostCert{}, fmt.Errorf("signing host cert with ca.key: %w", err)
+	}
+
+	hostKeyPEM := cert.MarshalX25519PrivateKey(hostKey)
+
+	hostCrtPEM, err := hostCrt.MarshalToPEM()
+	if err != nil {
+		return HostCert{}, fmt.Errorf("marshalling host.crt: %w", err)
+	}
+
+	return HostCert{
+		CACert:   caCrtPEM,
+		HostKey:  hostKeyPEM,
+		HostCert: hostCrtPEM,
+	}, nil
+}

go-workspace/src/tarutil/tgz_writer.go

@@ -7,6 +7,7 @@ import (
 	"crypto/sha512"
 	"fmt"
 	"io"
+	"io/fs"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -149,3 +150,22 @@ func (w *TGZWriter) WriteFileBytes(path string, body []byte) {
 	bodyR := bytes.NewReader(body)
 	w.WriteFile(path, bodyR.Size(), bodyR)
 }
+
+// CopyFileFromFS copies the file at the given path from srcFS into the same
+// path in the TGZWriter.
+func (w *TGZWriter) CopyFileFromFS(path string, srcFS fs.FS) error {
+
+	f, err := srcFS.Open(path)
+	if err != nil {
+		return fmt.Errorf("opening: %w", err)
+	}
+	defer f.Close()
+
+	fStat, err := f.Stat()
+	if err != nil {
+		return fmt.Errorf("stating: %w", err)
+	}
+
+	w.WriteFile(path, fStat.Size(), f)
+	return nil
+}