Remove hash.bin functionality from TGZWriter

dnsmasq/bin/dnsmasq-entrypoint

@@ -11,17 +11,17 @@ tmp="$(mktemp -d -t cryptic-net-dnsmasq-entrypoint-XXX)"
 
 ( trap "rm -rf '$tmp'" EXIT
 
-    tar xzf "$_BOOTSTRAP_PATH" -C "$tmp" ./nebula/hosts
+    tar xzf "$_BOOTSTRAP_PATH" -C "$tmp" ./hosts
 
     thisHostName=$(tar xzf "$_BOOTSTRAP_PATH" --to-stdout ./hostname)
-    thisHostIP=$(cat "$tmp"/nebula/hosts/"$thisHostName".yml | yq '.ip')
+    thisHostIP=$(cat "$tmp"/hosts/"$thisHostName".yml | yq '.nebula.ip')
 
     echo "listen-address=$thisHostIP" >> "$conf_path"
 
-    ls -1 "$tmp"/nebula/hosts | while read hostYml; do
+    ls -1 "$tmp"/hosts | while read hostYml; do
 
         hostName=$(echo "$hostYml" | cut -d. -f1)
-        hostIP=$(cat "$tmp"/nebula/hosts/"$hostYml" | yq '.ip')
+        hostIP=$(cat "$tmp"/hosts/"$hostYml" | yq '.nebula.ip')
         echo "address=/${hostName}.hosts.cryptic.io/$hostIP" >> "$conf_path"
 
     done

docs/dev/daemon-process-tree.plantuml

@@ -47,7 +47,7 @@ state AppDir {
     }
 
     state "./bin/garage -c $_RUNTIME_DIR_PATH/garage-N.toml server" as garage
-    state "./bin/cryptic-net-main garage-apply-layout-diff" as garageApplyLayoutDiff {
+    state "./bin/garage-apply-layout-diff" as garageApplyLayoutDiff {
         garageApplyLayoutDiff : * Runs once then exits
         garageApplyLayoutDiff : * Updates cluster topo
     }

garage/src/bin/garage-apply-layout-diff

@@ -5,10 +5,10 @@ tmp="$(mktemp -d -t cryptic-net-garage-apply-layout-diff-XXX)"
 
 ( trap "rm -rf '$tmp'" EXIT
 
-    tar xzf "$_BOOTSTRAP_PATH" -C "$tmp" ./nebula/hosts
+    tar xzf "$_BOOTSTRAP_PATH" -C "$tmp" ./hosts
 
     thisHostName=$(tar xzf "$_BOOTSTRAP_PATH" --to-stdout ./hostname)
-    thisHostIP=$(cat "$tmp"/nebula/hosts/"$thisHostName".yml | yq '.ip')
+    thisHostIP=$(cat "$tmp"/hosts/"$thisHostName".yml | yq '.nebula.ip')
 
     firstRPCPort=$(cat "$_DAEMON_YML_PATH" | yq '.storage.allocations[0].rpc_port')
 

go-workspace/src/admin/admin.go

@@ -0,0 +1,122 @@
+// Package admin deals with the parsing and creation of admin.tgz files.
+package admin
+
+import (
+	"cryptic-net/garage"
+	"cryptic-net/nebula"
+	"cryptic-net/tarutil"
+	"cryptic-net/yamlutil"
+	"fmt"
+	"io"
+	"io/fs"
+
+	"gopkg.in/yaml.v3"
+)
+
+const (
+	nebulaCertsCACertPath = "nebula/certs/ca.crt"
+	nebulaCertsCAKeyPath  = "nebula/certs/ca.key"
+
+	garageGlobalBucketKeyYmlPath = "garage/cryptic-net-global-bucket-key.yml"
+	garageAdminBucketKeyYmlPath  = "garage/cryptic-net-admin-bucket-key.yml"
+	garageRPCSecretPath          = "garage/rpc-secret.txt"
+)
+
+// Admin is used for accessing all information contained within an admin.tgz.
+type Admin struct {
+	NebulaCACert nebula.CACert
+
+	GarageRPCSecret                    string
+	GarageGlobalBucketS3APICredentials garage.S3APICredentials
+	GarageAdminBucketS3APICredentials  garage.S3APICredentials
+}
+
+// FromFS loads an Admin instance from the given fs.FS, which presumably
+// represents the file structure of an admin.tgz file.
+func FromFS(adminFS fs.FS) (Admin, error) {
+
+	var a Admin
+
+	filesToLoadAsYAML := []struct {
+		into interface{}
+		path string
+	}{
+		{&a.GarageGlobalBucketS3APICredentials, garageGlobalBucketKeyYmlPath},
+		{&a.GarageAdminBucketS3APICredentials, garageAdminBucketKeyYmlPath},
+	}
+
+	for _, f := range filesToLoadAsYAML {
+		if err := yamlutil.LoadYamlFSFile(f.into, adminFS, f.path); err != nil {
+			return Admin{}, fmt.Errorf("loading %q from fs: %w", f.path, err)
+		}
+	}
+
+	filesToLoadAsString := []struct {
+		into *string
+		path string
+	}{
+		{&a.NebulaCACert.CACert, nebulaCertsCACertPath},
+		{&a.NebulaCACert.CAKey, nebulaCertsCAKeyPath},
+		{&a.GarageRPCSecret, garageRPCSecretPath},
+	}
+
+	for _, f := range filesToLoadAsString {
+		body, err := fs.ReadFile(adminFS, f.path)
+		if err != nil {
+			return Admin{}, fmt.Errorf("loading %q from fs: %w", f.path, err)
+		}
+		*f.into = string(body)
+	}
+
+	return a, nil
+}
+
+// FromReader reads an admin.tgz file from the given io.Reader.
+func FromReader(r io.Reader) (Admin, error) {
+
+	fs, err := tarutil.FSFromReader(r)
+	if err != nil {
+		return Admin{}, fmt.Errorf("reading admin.tgz: %w", err)
+	}
+
+	return FromFS(fs)
+}
+
+// WriteTo writes the Admin as a new admin.tgz to the given io.Writer.
+func (a Admin) WriteTo(into io.Writer) error {
+
+	w := tarutil.NewTGZWriter(into)
+
+	filesToWriteAsYAML := []struct {
+		value interface{}
+		path  string
+	}{
+		{a.GarageGlobalBucketS3APICredentials, garageGlobalBucketKeyYmlPath},
+		{a.GarageAdminBucketS3APICredentials, garageAdminBucketKeyYmlPath},
+	}
+
+	for _, f := range filesToWriteAsYAML {
+
+		b, err := yaml.Marshal(f.value)
+		if err != nil {
+			return fmt.Errorf("yaml encoding data for %q: %w", f.path, err)
+		}
+
+		w.WriteFileBytes(f.path, b)
+	}
+
+	filesToWriteAsString := []struct {
+		value string
+		path  string
+	}{
+		{a.NebulaCACert.CACert, nebulaCertsCACertPath},
+		{a.NebulaCACert.CAKey, nebulaCertsCAKeyPath},
+		{a.GarageRPCSecret, garageRPCSecretPath},
+	}
+
+	for _, f := range filesToWriteAsString {
+		w.WriteFileBytes(f.path, []byte(f.value))
+	}
+
+	return w.Close()
+}

go-workspace/src/bootstrap/bootstrap.go

@@ -1,8 +1,10 @@
-// Package bootstrap deals with the creation of bootstrap files
+// Package bootstrap deals with the parsing and creation of bootstrap.tgz files.
+// It also contains some helpers which rely on bootstrap data.
 package bootstrap
 
 import (
 	"cryptic-net/garage"
+	"cryptic-net/nebula"
 	"cryptic-net/tarutil"
 	"cryptic-net/yamlutil"
 	"crypto/sha512"
@@ -12,7 +14,6 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
-	"strings"
 
 	"gopkg.in/yaml.v3"
 )
@@ -24,16 +25,11 @@ const (
 
 // Bootstrap is used for accessing all information contained within a
 // bootstrap.tgz file.
-//
-// An instance of Bootstrap is read-only, the creator sub-package should be used
-// to create new instances.
 type Bootstrap struct {
 	Hosts    map[string]Host
 	HostName string
 
-	NebulaCertsCACert   string
-	NebulaCertsHostCert string
-	NebulaCertsHostKey  string
+	NebulaHostCert nebula.HostCert
 
 	GarageRPCSecret                    string
 	GarageGlobalBucketS3APICredentials garage.S3APICredentials
@@ -65,9 +61,9 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
 		path string
 	}{
 		{&b.HostName, hostNamePath},
-		{&b.NebulaCertsCACert, nebulaCertsCACertPath},
-		{&b.NebulaCertsHostCert, nebulaCertsHostCertPath},
-		{&b.NebulaCertsHostKey, nebulaCertsHostKeyPath},
+		{&b.NebulaHostCert.CACert, nebulaCertsCACertPath},
+		{&b.NebulaHostCert.HostCert, nebulaCertsHostCertPath},
+		{&b.NebulaHostCert.HostKey, nebulaCertsHostKeyPath},
 		{&b.GarageRPCSecret, garageRPCSecretPath},
 	}
 
@@ -79,9 +75,6 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
 		*f.into = string(body)
 	}
 
-	// TODO confirm if this is necessary
-	b.GarageRPCSecret = strings.TrimSpace(b.GarageRPCSecret)
-
 	return b, nil
 }
 
@@ -118,9 +111,9 @@ func (b Bootstrap) WriteTo(into io.Writer) error {
 		path  string
 	}{
 		{b.HostName, hostNamePath},
-		{b.NebulaCertsCACert, nebulaCertsCACertPath},
-		{b.NebulaCertsHostCert, nebulaCertsHostCertPath},
-		{b.NebulaCertsHostKey, nebulaCertsHostKeyPath},
+		{b.NebulaHostCert.CACert, nebulaCertsCACertPath},
+		{b.NebulaHostCert.HostCert, nebulaCertsHostCertPath},
+		{b.NebulaHostCert.HostKey, nebulaCertsHostKeyPath},
 		{b.GarageRPCSecret, garageRPCSecretPath},
 	}
 

go-workspace/src/cmd/entrypoint/hosts.go

@@ -1,12 +1,11 @@
 package entrypoint
 
 import (
+	"cryptic-net/admin"
 	"cryptic-net/bootstrap"
 	"cryptic-net/nebula"
-	"cryptic-net/tarutil"
 	"errors"
 	"fmt"
-	"io/fs"
 	"net"
 	"os"
 	"regexp"
@@ -141,25 +140,25 @@ var subCmdHostsDelete = subCmd{
 	},
 }
 
-func readAdminFS(path string) (fs.FS, error) {
+func readAdmin(path string) (admin.Admin, error) {
 
 	if path == "-" {
 
-		outFS, err := tarutil.FSFromReader(os.Stdin)
+		adm, err := admin.FromReader(os.Stdin)
 		if err != nil {
-			return nil, fmt.Errorf("reading admin.tgz from stdin: %w", err)
+			return admin.Admin{}, fmt.Errorf("parsing admin.tgz from stdin: %w", err)
 		}
 
-		return outFS, nil
+		return adm, nil
 	}
 
 	f, err := os.Open(path)
 	if err != nil {
-		return nil, fmt.Errorf("opening file: %w", err)
+		return admin.Admin{}, fmt.Errorf("opening file: %w", err)
 	}
 	defer f.Close()
 
-	return tarutil.FSFromReader(f)
+	return admin.FromReader(f)
 }
 
 var subCmdHostsMakeBootstrap = subCmd{
@@ -190,7 +189,7 @@ var subCmdHostsMakeBootstrap = subCmd{
 
 		env := subCmdCtx.env
 
-		adminFS, err := readAdminFS(*adminPath)
+		adm, err := readAdmin(*adminPath)
 		if err != nil {
 			return fmt.Errorf("reading admin.tgz with --admin-path of %q: %w", *adminPath, err)
 		}
@@ -214,7 +213,7 @@ var subCmdHostsMakeBootstrap = subCmd{
 			return fmt.Errorf("couldn't find host into for %q in garage, has `cryptic-net hosts add` been run yet?", *name)
 		}
 
-		nebulaHostCert, err := nebula.NewHostCert(adminFS, host)
+		nebulaHostCert, err := nebula.NewHostCert(adm.NebulaCACert, host.Name, host.Nebula.IP)
 		if err != nil {
 			return fmt.Errorf("creating new nebula host key/cert: %w", err)
 		}
@@ -223,13 +222,10 @@ var subCmdHostsMakeBootstrap = subCmd{
 			Hosts:    hosts,
 			HostName: *name,
 
-			NebulaCertsCACert:   nebulaHostCert.CACert,
-			NebulaCertsHostCert: nebulaHostCert.HostCert,
-			NebulaCertsHostKey:  nebulaHostCert.HostKey,
+			NebulaHostCert: nebulaHostCert,
 
-			// TODO these should use adminFS
-			GarageRPCSecret:                    env.Bootstrap.GarageRPCSecret,
-			GarageGlobalBucketS3APICredentials: env.Bootstrap.GarageGlobalBucketS3APICredentials,
+			GarageRPCSecret:                    adm.GarageRPCSecret,
+			GarageGlobalBucketS3APICredentials: adm.GarageGlobalBucketS3APICredentials,
 		}
 
 		return newBootstrap.WriteTo(os.Stdout)

go-workspace/src/cmd/nebula-entrypoint/main.go

@@ -37,9 +37,9 @@ func Main() {
 
 	config := map[string]interface{}{
 		"pki": map[string]string{
-			"ca":   env.Bootstrap.NebulaCertsCACert,
-			"cert": env.Bootstrap.NebulaCertsHostCert,
-			"key":  env.Bootstrap.NebulaCertsHostKey,
+			"ca":   env.Bootstrap.NebulaHostCert.CACert,
+			"cert": env.Bootstrap.NebulaHostCert.HostCert,
+			"key":  env.Bootstrap.NebulaHostCert.HostKey,
 		},
 		"static_host_map": staticHostMap,
 		"punchy": map[string]bool{

go-workspace/src/nebula/nebula.go

@@ -3,12 +3,10 @@
 package nebula
 
 import (
-	"cryptic-net/bootstrap"
 	"crypto/ed25519"
 	"crypto/rand"
 	"fmt"
 	"io"
-	"io/fs"
 	"net"
 	"time"
 
@@ -43,7 +41,7 @@ type CACert struct {
 // NewHostCert generates a new key/cert for a nebula host using the CA key
 // which will be found in the adminFS.
 func NewHostCert(
-	adminFS fs.FS, host bootstrap.Host,
+	caCert CACert, hostName, hostIP string,
 ) (
 	HostCert, error,
 ) {
@@ -51,22 +49,12 @@ func NewHostCert(
 	// The logic here is largely based on
 	// https://github.com/slackhq/nebula/blob/v1.4.0/cmd/nebula-cert/sign.go
 
-	caKeyPEM, err := fs.ReadFile(adminFS, "nebula/certs/ca.key")
-	if err != nil {
-		return HostCert{}, fmt.Errorf("reading ca.key from admin fs: %w", err)
-	}
-
-	caKey, _, err := cert.UnmarshalEd25519PrivateKey(caKeyPEM)
+	caKey, _, err := cert.UnmarshalEd25519PrivateKey([]byte(caCert.CAKey))
 	if err != nil {
 		return HostCert{}, fmt.Errorf("unmarshaling ca.key: %w", err)
 	}
 
-	caCrtPEM, err := fs.ReadFile(adminFS, "nebula/certs/ca.crt")
-	if err != nil {
-		return HostCert{}, fmt.Errorf("reading ca.crt from admin fs: %w", err)
-	}
-
-	caCrt, _, err := cert.UnmarshalNebulaCertificateFromPEM(caCrtPEM)
+	caCrt, _, err := cert.UnmarshalNebulaCertificateFromPEM([]byte(caCert.CACert))
 	if err != nil {
 		return HostCert{}, fmt.Errorf("unmarshaling ca.crt: %w", err)
 	}
@@ -78,9 +66,9 @@ func NewHostCert(
 
 	expireAt := caCrt.Details.NotAfter.Add(-1 * time.Second)
 
-	ip := net.ParseIP(host.Nebula.IP)
+	ip := net.ParseIP(hostIP)
 	if ip == nil {
-		return HostCert{}, fmt.Errorf("invalid host ip %q", host.Nebula.IP)
+		return HostCert{}, fmt.Errorf("invalid host ip %q", hostIP)
 	}
 
 	ipNet := &net.IPNet{
@@ -100,7 +88,7 @@ func NewHostCert(
 
 	hostCrt := cert.NebulaCertificate{
 		Details: cert.NebulaCertificateDetails{
-			Name:      host.Name,
+			Name:      hostName,
 			Ips:       []*net.IPNet{ipNet},
 			NotBefore: time.Now(),
 			NotAfter:  expireAt,
@@ -126,7 +114,7 @@ func NewHostCert(
 	}
 
 	return HostCert{
-		CACert:   string(caCrtPEM),
+		CACert:   caCert.CACert,
 		HostKey:  string(hostKeyPEM),
 		HostCert: string(hostCrtPEM),
 	}, nil

go-workspace/src/tarutil/tgz_writer.go

@@ -4,40 +4,21 @@ import (
 	"archive/tar"
 	"bytes"
 	"compress/gzip"
-	"crypto/sha512"
 	"fmt"
 	"io"
-	"io/fs"
 	"path/filepath"
-	"sort"
 	"strings"
 )
 
-const (
-
-	// Path to the file containing the content hash of the tgz, which is
-	// included as part of all tgz files created by TGZWriter.
-	HashBinPath = "hash.bin"
-)
-
-type fileHash struct {
-	path string
-	hash []byte
-}
-
 // TGZWriter is a utility for writing tgz files. If an internal error is
 // encountered by any method then all subsequent methods will be no-ops, and
 // Close() will return that error (after closing out resources).
-//
-// A `hash.bin` file will be automatically included in the resulting tgz, which
-// will contain a consistent hash of all other contents in the tgz file.
 type TGZWriter struct {
 	gzipW *gzip.Writer
 	tarW  *tar.Writer
 	err   error
 
 	dirsWritten map[string]bool
-	fileHashes  []fileHash
 }
 
 // NewTGZWriter initializes and returns a new instance of TGZWriter which will
@@ -55,22 +36,8 @@ func NewTGZWriter(w io.Writer) *TGZWriter {
 // Close cleans up all open resources being held by TGZWriter, and returns the
 // first internal error which was encountered during its operation (if any).
 func (w *TGZWriter) Close() error {
-
-	sort.Slice(w.fileHashes, func(i, j int) bool {
-		return w.fileHashes[i].path < w.fileHashes[j].path
-	})
-
-	h := sha512.New()
-
-	for i := range w.fileHashes {
-		fmt.Fprintf(h, "%q:%x\n", w.fileHashes[i].path, w.fileHashes[i].hash)
-	}
-
-	w.WriteFile(HashBinPath, int64(h.Size()), bytes.NewBuffer(h.Sum(nil)))
-
 	w.tarW.Close()
 	w.gzipW.Close()
-
 	return w.err
 }
 
@@ -131,17 +98,10 @@ func (w *TGZWriter) WriteFile(path string, size int64, body io.Reader) {
 		return
 	}
 
-	h := sha512.New()
-
-	if _, err := io.Copy(io.MultiWriter(w.tarW, h), body); err != nil {
+	if _, err := io.Copy(w.tarW, body); err != nil {
 		w.err = fmt.Errorf("writing file body of file %q: %w", path, err)
 		return
 	}
-
-	w.fileHashes = append(w.fileHashes, fileHash{
-		path: path,
-		hash: h.Sum(nil),
-	})
 }
 
 // WriteFileBytes is a shortcut for calling WriteFile with the given byte slice
@@ -150,22 +110,3 @@ func (w *TGZWriter) WriteFileBytes(path string, body []byte) {
 	bodyR := bytes.NewReader(body)
 	w.WriteFile(path, bodyR.Size(), bodyR)
 }
-
-// CopyFileFromFS copies the file at the given path from srcFS into the same
-// path in the TGZWriter.
-func (w *TGZWriter) CopyFileFromFS(path string, srcFS fs.FS) error {
-
-	f, err := srcFS.Open(path)
-	if err != nil {
-		return fmt.Errorf("opening: %w", err)
-	}
-	defer f.Close()
-
-	fStat, err := f.Stat()
-	if err != nil {
-		return fmt.Errorf("stating: %w", err)
-	}
-
-	w.WriteFile(path, fStat.Size(), f)
-	return nil
-}