Factor out garage-apply-layout-diff

The new code runs the equivalent functionality within the daemon go
code. It was required to make Env be immutable in order to prevent race
conditions (this really should have been done from the beginning
anyway).

garage/default.nix

@@ -26,7 +26,6 @@ in rec {
     paths = [
       garage
       minioClient
-      ./src
     ];
   };
 

garage/src/bin/garage-apply-layout-diff

@@ -1,24 +0,0 @@
-
-set -e -o pipefail
-
-tmp="$(mktemp -d -t cryptic-net-garage-apply-layout-diff-XXX)"
-
-( trap "rm -rf '$tmp'" EXIT
-
-    tar xzf "$_BOOTSTRAP_PATH" -C "$tmp" ./hosts
-
-    thisHostName=$(tar xzf "$_BOOTSTRAP_PATH" --to-stdout ./hostname)
-    thisHostIP=$(cat "$tmp"/hosts/"$thisHostName".yml | yq '.nebula.ip')
-
-    firstRPCPort=$(cat "$_DAEMON_YML_PATH" | yq '.storage.allocations[0].rpc_port')
-
-    firstPeerID=$(cryptic-net-main garage-peer-keygen -danger -ip "$thisHostIP" -port "$firstRPCPort")
-
-    export GARAGE_RPC_HOST="$firstPeerID"@"$thisHostIP":"$firstRPCPort"
-    export GARAGE_RPC_SECRET=$(tar -xzf "$_BOOTSTRAP_PATH" --to-stdout "./garage/rpc-secret.txt")
-
-    garage layout show | cryptic-net-main garage-layout-diff | while read diffLine; do
-        echo "> $diffLine"
-        $diffLine
-    done
-)

go-workspace/src/admin/admin.go

@@ -27,8 +27,8 @@ const (
 // CreationParams are general parameters used when creating a new network. These
 // are available to all hosts within the network via their bootstrap files.
 type CreationParams struct {
+	ID     string `yaml:"id"`
 	Domain string `yaml:"domain"`
-	CIDRs  []string `yaml:"cidrs"`
 }
 
 // Admin is used for accessing all information contained within an admin.tgz.

go-workspace/src/cmd/cryptic-net-main/main.go

@@ -16,8 +16,6 @@ package main
 
 import (
 	"cryptic-net/cmd/entrypoint"
-	garage_layout_diff "cryptic-net/cmd/garage-layout-diff"
-	garage_peer_keygen "cryptic-net/cmd/garage-peer-keygen"
 	nebula_entrypoint "cryptic-net/cmd/nebula-entrypoint"
 	update_global_bucket "cryptic-net/cmd/update-global-bucket"
 	"fmt"
@@ -31,8 +29,6 @@ type mainFn struct {
 
 var mainFns = []mainFn{
 	{"entrypoint", entrypoint.Main},
-	{"garage-layout-diff", garage_layout_diff.Main},
-	{"garage-peer-keygen", garage_peer_keygen.Main},
 	{"nebula-entrypoint", nebula_entrypoint.Main},
 	{"update-global-bucket", update_global_bucket.Main},
 }

go-workspace/src/cmd/entrypoint/admin.go

@@ -1,14 +1,20 @@
 package entrypoint
 
 import (
+	"context"
 	"cryptic-net/admin"
 	"cryptic-net/bootstrap"
+	"cryptic-net/garage"
 	"cryptic-net/nebula"
 	"crypto/rand"
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"net"
 	"os"
+	"strings"
+
+	"github.com/cryptic-io/pmux/pmuxlib"
 )
 
 func randStr(l int) string {
@@ -40,6 +46,206 @@ func readAdmin(path string) (admin.Admin, error) {
 	return admin.FromReader(f)
 }
 
+var subCmdAdminCreateNetwork = subCmd{
+	name:  "create-network",
+	descr: "Creates a new cryptic-net network, outputting the resulting admin.tgz to stdout",
+	do: func(subCmdCtx subCmdCtx) error {
+
+		flags := subCmdCtx.flagSet(false)
+
+		daemonYmlPath := flags.StringP(
+			"config-path", "c", "",
+			"Optional path to a daemon.yml file to load configuration from.",
+		)
+
+		dumpConfig := flags.Bool(
+			"dump-config", false,
+			"Write the default configuration file to stdout and exit.",
+		)
+
+		domain := flags.StringP(
+			"domain", "d", "",
+			"Domain name that should be used as the root domain in the network.",
+		)
+
+		subnetStr := flags.StringP(
+			"subnet", "s", "",
+			"CIDR which denotes the subnet that IPs hosts on the network can be assigned.",
+		)
+
+		hostName := flags.StringP(
+			"name", "n", "",
+			"Name of the host which will be the first host in the network",
+		)
+
+		if err := flags.Parse(subCmdCtx.args); err != nil {
+			return fmt.Errorf("parsing flags: %w", err)
+		}
+
+		env := subCmdCtx.env
+
+		if *dumpConfig {
+			return writeBuiltinDaemonYml(env, os.Stdout)
+		}
+
+		if *domain == "" || *subnetStr == "" || *hostName == "" {
+			return errors.New("--domain, --subnet, and --name are required")
+		}
+
+		*domain = strings.TrimRight(strings.TrimLeft(*domain, "."), ".")
+
+		ip, subnet, err := net.ParseCIDR(*subnetStr)
+		if err != nil {
+			return fmt.Errorf("parsing %q as a CIDR: %w", *subnetStr, err)
+		}
+
+		if err := validateHostName(*hostName); err != nil {
+			return fmt.Errorf("invalid hostname %q: %w", *hostName, err)
+		}
+
+		adminCreationParams := admin.CreationParams{
+			ID:     randStr(32),
+			Domain: *domain,
+		}
+
+		{
+			runtimeDirPath := env.RuntimeDirPath
+
+			fmt.Fprintf(os.Stderr, "will use runtime directory %q for temporary state\n", runtimeDirPath)
+
+			if err := os.MkdirAll(runtimeDirPath, 0700); err != nil {
+				return fmt.Errorf("creating directory %q: %w", runtimeDirPath, err)
+			}
+
+			defer func() {
+				fmt.Fprintf(os.Stderr, "cleaning up runtime directory %q\n", runtimeDirPath)
+				if err := os.RemoveAll(runtimeDirPath); err != nil {
+					fmt.Fprintf(os.Stderr, "error removing temporary directory %q: %v", runtimeDirPath, err)
+				}
+			}()
+		}
+
+		if err := writeMergedDaemonYml(env, *daemonYmlPath); err != nil {
+			return fmt.Errorf("merging and writing daemon.yml file: %w", err)
+		}
+
+		daemon := env.ThisDaemon()
+
+		if len(daemon.Storage.Allocations) < 3 {
+			return fmt.Errorf("daemon.yml with at least 3 allocations was not provided")
+		}
+
+		nebulaCACert, err := nebula.NewCACert(*domain, subnet)
+		if err != nil {
+			return fmt.Errorf("creating nebula CA cert: %w", err)
+		}
+
+		nebulaHostCert, err := nebula.NewHostCert(nebulaCACert, *hostName, ip)
+		if err != nil {
+			return fmt.Errorf("creating nebula cert for host: %w", err)
+		}
+
+		env.Bootstrap = bootstrap.Bootstrap{
+			AdminCreationParams: adminCreationParams,
+			Hosts: map[string]bootstrap.Host{
+				*hostName: bootstrap.Host{
+					Name: *hostName,
+					Nebula: bootstrap.NebulaHost{
+						IP: ip.String(),
+					},
+				},
+			},
+			HostName:                           *hostName,
+			NebulaHostCert:                     nebulaHostCert,
+			GarageRPCSecret:                    randStr(32),
+			GarageGlobalBucketS3APICredentials: garage.NewS3APICredentials(),
+		}
+
+		if env, err = mergeDaemonIntoBootstrap(env); err != nil {
+			return fmt.Errorf("merging daemon.yml into bootstrap data: %w", err)
+		}
+
+		// TODO this can be gotten rid of once nebula-entrypoint is rolled into
+		// daemon itself
+		for key, val := range env.ToMap() {
+			if err := os.Setenv(key, val); err != nil {
+				return fmt.Errorf("failed to set %q to %q: %w", key, val, err)
+			}
+		}
+
+		garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env)
+		if err != nil {
+			return fmt.Errorf("generating garage children configs: %w", err)
+		}
+
+		pmuxConfig := pmuxlib.Config{
+			Processes: append(
+				[]pmuxlib.ProcessConfig{
+					nebulaEntrypointPmuxProcConfig(),
+				},
+				garageChildrenPmuxProcConfigs...,
+			),
+		}
+
+		ctx, cancel := context.WithCancel(env.Context)
+		pmuxDoneCh := make(chan struct{})
+
+		fmt.Fprintln(os.Stderr, "starting child processes")
+		go func() {
+			pmuxlib.Run(ctx, pmuxConfig)
+			close(pmuxDoneCh)
+		}()
+
+		defer func() {
+			cancel()
+			fmt.Fprintln(os.Stderr, "waiting for child processes to exit")
+			<-pmuxDoneCh
+		}()
+
+		fmt.Fprintln(os.Stderr, "waiting for garage instances to come online")
+		if err := waitForGarage(ctx, env); err != nil {
+			return fmt.Errorf("waiting for garage to start up: %w", err)
+		}
+
+		fmt.Fprintln(os.Stderr, "applying initial garage layout")
+		if err := garageApplyLayout(ctx, env); err != nil {
+			return fmt.Errorf("applying initial garage layout: %w", err)
+		}
+
+		fmt.Fprintln(os.Stderr, "initializing garage shared global bucket")
+		if err := garageInitializeGlobalBucket(ctx, env); err != nil {
+			return fmt.Errorf("initializing garage shared global bucket: %w", err)
+		}
+
+		garageS3Client, err := env.Bootstrap.GlobalBucketS3APIClient()
+		if err != nil {
+			return fmt.Errorf("initializing garage shared global bucket client: %w", err)
+		}
+
+		fmt.Fprintln(os.Stderr, "writing data for this host into garage")
+		err = bootstrap.PutGarageBoostrapHost(ctx, garageS3Client, env.Bootstrap.ThisHost())
+		if err != nil {
+			return fmt.Errorf("putting host data into garage: %w", err)
+		}
+
+		fmt.Fprintln(os.Stderr, "cluster initialized successfully, writing admin.tgz to stdout")
+
+		err = admin.Admin{
+			CreationParams:                     adminCreationParams,
+			NebulaCACert:                       nebulaCACert,
+			GarageRPCSecret:                    env.Bootstrap.GarageRPCSecret,
+			GarageGlobalBucketS3APICredentials: env.Bootstrap.GarageGlobalBucketS3APICredentials,
+			GarageAdminBucketS3APICredentials:  garage.NewS3APICredentials(),
+		}.WriteTo(os.Stdout)
+
+		if err != nil {
+			return fmt.Errorf("writing admin.tgz to stdout")
+		}
+
+		return nil
+	},
+}
+
 var subCmdAdminMakeBootstrap = subCmd{
 	name:      "make-bootstrap",
 	descr:     "Creates a new bootstrap.tgz file for a particular host and writes it to stdout",
@@ -92,7 +298,12 @@ var subCmdAdminMakeBootstrap = subCmd{
 			return fmt.Errorf("couldn't find host into for %q in garage, has `cryptic-net hosts add` been run yet?", *name)
 		}
 
-		nebulaHostCert, err := nebula.NewHostCert(adm.NebulaCACert, host.Name, host.Nebula.IP)
+		ip := net.ParseIP(host.Nebula.IP)
+		if ip == nil {
+			return fmt.Errorf("invalid IP stored with host %q: %q", *name, host.Nebula.IP)
+		}
+
+		nebulaHostCert, err := nebula.NewHostCert(adm.NebulaCACert, host.Name, ip)
 		if err != nil {
 			return fmt.Errorf("creating new nebula host key/cert: %w", err)
 		}

go-workspace/src/cmd/entrypoint/daemon.go

@@ -40,44 +40,44 @@ import (
 
 // creates a new bootstrap file using available information from the network. If
 // the new bootstrap file is different than the existing one, the existing one
-// is overwritten, ReloadBootstrap is called on env, true is returned.
-func reloadBootstrap(env *crypticnet.Env, s3Client garage.S3APIClient) (bool, error) {
+// is overwritten, env's bootstrap is reloaded, true is returned.
+func reloadBootstrap(env crypticnet.Env, s3Client garage.S3APIClient) (crypticnet.Env, bool, error) {
 
 	newHosts, err := bootstrap.GetGarageBootstrapHosts(env.Context, s3Client)
 	if err != nil {
-		return false, fmt.Errorf("getting hosts from garage: %w", err)
+		return crypticnet.Env{}, false, fmt.Errorf("getting hosts from garage: %w", err)
 	}
 
 	newHostsHash, err := bootstrap.HostsHash(newHosts)
 	if err != nil {
-		return false, fmt.Errorf("calculating hash of new hosts: %w", err)
+		return crypticnet.Env{}, false, fmt.Errorf("calculating hash of new hosts: %w", err)
 	}
 
 	currHostsHash, err := bootstrap.HostsHash(env.Bootstrap.Hosts)
 	if err != nil {
-		return false, fmt.Errorf("calculating hash of current hosts: %w", err)
+		return crypticnet.Env{}, false, fmt.Errorf("calculating hash of current hosts: %w", err)
 	}
 
 	if bytes.Equal(newHostsHash, currHostsHash) {
-		return false, nil
+		return crypticnet.Env{}, false, nil
 	}
 
 	buf := new(bytes.Buffer)
 	if err := env.Bootstrap.WithHosts(newHosts).WriteTo(buf); err != nil {
-		return false, fmt.Errorf("writing new bootstrap file to buffer: %w", err)
+		return crypticnet.Env{}, false, fmt.Errorf("writing new bootstrap file to buffer: %w", err)
 	}
 
-	if err := copyBootstrapToDataDir(env, buf); err != nil {
-		return false, fmt.Errorf("copying new bootstrap file to data dir: %w", err)
+	if env, err = copyBootstrapToDataDirAndReload(env, buf); err != nil {
+		return crypticnet.Env{}, false, fmt.Errorf("copying new bootstrap file to data dir: %w", err)
 	}
 
-	return true, nil
+	return env, true, nil
 }
 
 // runs a single pmux process ofor daemon, returning only once the env.Context
 // has been canceled or bootstrap info has been changed. This will always block
 // until the spawned pmux has returned.
-func runDaemonPmuxOnce(env *crypticnet.Env, s3Client garage.S3APIClient) error {
+func runDaemonPmuxOnce(env crypticnet.Env, s3Client garage.S3APIClient) error {
 
 	thisHost := env.Bootstrap.ThisHost()
 	thisDaemon := env.ThisDaemon()
@@ -100,7 +100,6 @@ func runDaemonPmuxOnce(env *crypticnet.Env, s3Client garage.S3APIClient) error {
 		}
 
 		pmuxProcConfigs = append(pmuxProcConfigs, garageChildrenPmuxProcConfigs...)
-		pmuxProcConfigs = append(pmuxProcConfigs, garageApplyLayoutDiffPmuxProcConfig(env))
 	}
 
 	pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
@@ -126,6 +125,26 @@ func runDaemonPmuxOnce(env *crypticnet.Env, s3Client garage.S3APIClient) error {
 		pmuxlib.Run(ctx, pmuxConfig)
 	}()
 
+	if len(thisDaemon.Storage.Allocations) > 0 {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+
+			if err := waitForGarage(ctx, env); err != nil {
+				fmt.Fprintf(os.Stderr, "aborted waiting for garage instances to start: %v\n", err)
+				return
+			}
+
+			err := doOnce(ctx, func(ctx context.Context) error {
+				return garageApplyLayout(ctx, env)
+			})
+
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "aborted applying garage layout: %v\n", err)
+			}
+		}()
+	}
+
 	ticker := time.NewTicker(3 * time.Minute)
 	defer ticker.Stop()
 
@@ -139,7 +158,12 @@ func runDaemonPmuxOnce(env *crypticnet.Env, s3Client garage.S3APIClient) error {
 
 			fmt.Fprintln(os.Stderr, "checking for changes to bootstrap")
 
-			if changed, err := reloadBootstrap(env, s3Client); err != nil {
+			var (
+				changed bool
+				err     error
+			)
+
+			if env, changed, err = reloadBootstrap(env, s3Client); err != nil {
 				return fmt.Errorf("reloading bootstrap: %w", err)
 
 			} else if changed {
@@ -226,7 +250,7 @@ var subCmdDaemon = subCmd{
 				return fmt.Errorf("opening file %q: %w", env.BootstrapPath, err)
 			}
 
-			err = copyBootstrapToDataDir(env, f)
+			env, err = copyBootstrapToDataDirAndReload(env, f)
 			f.Close()
 
 			if err != nil {
@@ -238,12 +262,14 @@ var subCmdDaemon = subCmd{
 			return fmt.Errorf("merging and writing daemon.yml file: %w", err)
 		}
 
+		var err error
+
 		// we update this Host's data using whatever configuration has been
 		// provided by daemon.yml. This way the daemon has the most
 		// up-to-date possible bootstrap. This updated bootstrap will later
 		// get updated in garage using update-global-bucket, so other hosts
 		// will see it as well.
-		if err := mergeDaemonIntoBootstrap(env); err != nil {
+		if env, err = mergeDaemonIntoBootstrap(env); err != nil {
 			return fmt.Errorf("merging daemon.yml into bootstrap data: %w", err)
 		}
 

go-workspace/src/cmd/entrypoint/daemon_util.go

@@ -2,49 +2,42 @@ package entrypoint
 
 import (
 	"bytes"
+	"context"
 	crypticnet "cryptic-net"
 	"cryptic-net/bootstrap"
-	"cryptic-net/garage"
 	"fmt"
 	"io"
-	"net"
 	"os"
 	"path/filepath"
-	"strconv"
-	"time"
 
 	"github.com/cryptic-io/pmux/pmuxlib"
 )
 
-func copyBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
+func copyBootstrapToDataDirAndReload(env crypticnet.Env, r io.Reader) (crypticnet.Env, error) {
 
 	path := env.DataDirBootstrapPath()
 	dirPath := filepath.Dir(path)
 
 	if err := os.MkdirAll(dirPath, 0700); err != nil {
-		return fmt.Errorf("creating directory %q: %w", dirPath, err)
+		return crypticnet.Env{}, fmt.Errorf("creating directory %q: %w", dirPath, err)
 	}
 
 	f, err := os.Create(path)
 	if err != nil {
-		return fmt.Errorf("creating file %q: %w", path, err)
+		return crypticnet.Env{}, fmt.Errorf("creating file %q: %w", path, err)
 	}
 
 	_, err = io.Copy(f, r)
 	f.Close()
 
 	if err != nil {
-		return fmt.Errorf("copying bootstrap file to %q: %w", path, err)
+		return crypticnet.Env{}, fmt.Errorf("copying bootstrap file to %q: %w", path, err)
 	}
 
-	if err := env.LoadBootstrap(path); err != nil {
-		return fmt.Errorf("loading bootstrap from %q: %w", path, err)
-	}
-
-	return nil
+	return env.LoadBootstrap(path)
 }
 
-func mergeDaemonIntoBootstrap(env *crypticnet.Env) error {
+func mergeDaemonIntoBootstrap(env crypticnet.Env) (crypticnet.Env, error) {
 	daemon := env.ThisDaemon()
 	host := env.Bootstrap.ThisHost()
 
@@ -68,44 +61,27 @@ func mergeDaemonIntoBootstrap(env *crypticnet.Env) error {
 
 	buf := new(bytes.Buffer)
 	if err := env.Bootstrap.WithHosts(env.Bootstrap.Hosts).WriteTo(buf); err != nil {
-		return fmt.Errorf("writing new bootstrap file to buffer: %w", err)
+		return crypticnet.Env{}, fmt.Errorf("writing new bootstrap file to buffer: %w", err)
 	}
 
-	if err := copyBootstrapToDataDir(env, buf); err != nil {
-		return fmt.Errorf("copying new bootstrap file to data dir: %w", err)
-	}
-
-	return nil
+	return copyBootstrapToDataDirAndReload(env, buf)
 }
 
-func waitForNebulaArgs(env *crypticnet.Env, args ...string) []string {
+func doOnce(ctx context.Context, fn func(context.Context) error) error {
+	for {
+		if err := fn(ctx); err == nil {
+			return nil
+		} else if ctxErr := ctx.Err(); ctxErr != nil {
+			return ctxErr
+		}
+	}
+}
+
+func waitForNebulaArgs(env crypticnet.Env, args ...string) []string {
 	thisHost := env.Bootstrap.ThisHost()
 	return append([]string{"wait-for-ip", thisHost.Nebula.IP}, args...)
 }
 
-func waitForGarageArgs(env *crypticnet.Env, args ...string) []string {
-
-	thisHost := env.Bootstrap.ThisHost()
-	allocs := env.ThisDaemon().Storage.Allocations
-
-	if len(allocs) == 0 {
-		return waitForNebulaArgs(env, args...)
-	}
-
-	var preArgs []string
-
-	for _, alloc := range allocs {
-		preArgs = append(
-			preArgs,
-			"wait-for",
-			net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
-			"--",
-		)
-	}
-
-	return append(preArgs, args...)
-}
-
 func nebulaEntrypointPmuxProcConfig() pmuxlib.ProcessConfig {
 	return pmuxlib.ProcessConfig{
 		Name: "nebula",
@@ -115,91 +91,3 @@ func nebulaEntrypointPmuxProcConfig() pmuxlib.ProcessConfig {
 		},
 	}
 }
-
-func garageWriteChildConf(
-	env *crypticnet.Env,
-	alloc crypticnet.DaemonYmlStorageAllocation,
-) (
-	string, error,
-) {
-
-	if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
-		return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
-	}
-
-	thisHost := env.Bootstrap.ThisHost()
-
-	peer := garage.Peer{
-		IP:        thisHost.Nebula.IP,
-		RPCPort:   alloc.RPCPort,
-		S3APIPort: alloc.S3APIPort,
-	}
-
-	pubKey, privKey := peer.RPCPeerKey()
-
-	nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
-	nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
-
-	if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
-		return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
-
-	} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
-		return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
-	}
-
-	garageTomlPath := filepath.Join(
-		env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
-	)
-
-	err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
-		MetaPath: alloc.MetaPath,
-		DataPath: alloc.DataPath,
-
-		RPCSecret:  env.Bootstrap.GarageRPCSecret,
-		AdminToken: env.Bootstrap.GarageAdminToken,
-
-		RPCAddr:   net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
-		APIAddr:   net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
-		AdminAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.AdminPort)),
-
-		BootstrapPeers: env.Bootstrap.GarageRPCPeerAddrs(),
-	})
-
-	if err != nil {
-		return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
-	}
-
-	return garageTomlPath, nil
-}
-
-func garageChildrenPmuxProcConfigs(env *crypticnet.Env) ([]pmuxlib.ProcessConfig, error) {
-
-	var pmuxProcConfigs []pmuxlib.ProcessConfig
-
-	for _, alloc := range env.ThisDaemon().Storage.Allocations {
-
-		childConfPath, err := garageWriteChildConf(env, alloc)
-
-		if err != nil {
-			return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
-		}
-
-		pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
-			Name:        fmt.Sprintf("garage-%d", alloc.RPCPort),
-			Cmd:         "garage",
-			Args:        []string{"-c", childConfPath, "server"},
-			SigKillWait: 1 * time.Minute,
-		})
-	}
-
-	return pmuxProcConfigs, nil
-}
-
-func garageApplyLayoutDiffPmuxProcConfig(env *crypticnet.Env) pmuxlib.ProcessConfig {
-	return pmuxlib.ProcessConfig{
-		Name:        "garage-apply-layout-diff",
-		Cmd:         "bash",
-		Args:        waitForGarageArgs(env, "bash", "garage-apply-layout-diff"),
-		NoRestartOn: []int{0},
-	}
-}

go-workspace/src/cmd/entrypoint/daemon_yml.go

@@ -13,11 +13,11 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-func builtinDaemonYmlPath(env *crypticnet.Env) string {
+func builtinDaemonYmlPath(env crypticnet.Env) string {
 	return filepath.Join(env.AppDirPath, "etc", "daemon.yml")
 }
 
-func writeBuiltinDaemonYml(env *crypticnet.Env, w io.Writer) error {
+func writeBuiltinDaemonYml(env crypticnet.Env, w io.Writer) error {
 
 	builtinDaemonYmlPath := builtinDaemonYmlPath(env)
 
@@ -33,7 +33,7 @@ func writeBuiltinDaemonYml(env *crypticnet.Env, w io.Writer) error {
 	return nil
 }
 
-func writeMergedDaemonYml(env *crypticnet.Env, userDaemonYmlPath string) error {
+func writeMergedDaemonYml(env crypticnet.Env, userDaemonYmlPath string) error {
 
 	builtinDaemonYmlPath := builtinDaemonYmlPath(env)
 

go-workspace/src/cmd/entrypoint/garage_util.go

@@ -0,0 +1,273 @@
+package entrypoint
+
+import (
+	"context"
+	crypticnet "cryptic-net"
+	"cryptic-net/garage"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/cryptic-io/pmux/pmuxlib"
+)
+
+func waitForGarage(ctx context.Context, env crypticnet.Env) error {
+
+	for _, alloc := range env.ThisDaemon().Storage.Allocations {
+
+		adminAddr := net.JoinHostPort(
+			env.Bootstrap.ThisHost().Nebula.IP,
+			strconv.Itoa(alloc.AdminPort),
+		)
+
+		adminClient := garage.NewAdminClient(
+			adminAddr,
+			env.Bootstrap.GarageAdminToken,
+		)
+
+		if err := adminClient.Wait(ctx); err != nil {
+			return fmt.Errorf("waiting for instance %q to start up: %w", adminAddr, err)
+		}
+	}
+
+	return nil
+
+}
+
+func waitForGarageArgs(env crypticnet.Env, args ...string) []string {
+
+	thisHost := env.Bootstrap.ThisHost()
+	allocs := env.ThisDaemon().Storage.Allocations
+
+	if len(allocs) == 0 {
+		return waitForNebulaArgs(env, args...)
+	}
+
+	var preArgs []string
+
+	for _, alloc := range allocs {
+		preArgs = append(
+			preArgs,
+			"wait-for",
+			net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
+			"--",
+		)
+	}
+
+	return append(preArgs, args...)
+}
+
+func garageWriteChildConf(
+	env crypticnet.Env,
+	alloc crypticnet.DaemonYmlStorageAllocation,
+) (
+	string, error,
+) {
+
+	if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
+		return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
+	}
+
+	thisHost := env.Bootstrap.ThisHost()
+
+	peer := garage.Peer{
+		IP:        thisHost.Nebula.IP,
+		RPCPort:   alloc.RPCPort,
+		S3APIPort: alloc.S3APIPort,
+	}
+
+	pubKey, privKey := peer.RPCPeerKey()
+
+	nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
+	nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
+
+	if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
+		return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
+
+	} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
+		return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
+	}
+
+	garageTomlPath := filepath.Join(
+		env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
+	)
+
+	err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
+		MetaPath: alloc.MetaPath,
+		DataPath: alloc.DataPath,
+
+		RPCSecret:  env.Bootstrap.GarageRPCSecret,
+		AdminToken: env.Bootstrap.GarageAdminToken,
+
+		RPCAddr:   net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
+		APIAddr:   net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
+		AdminAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.AdminPort)),
+
+		BootstrapPeers: env.Bootstrap.GarageRPCPeerAddrs(),
+	})
+
+	if err != nil {
+		return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
+	}
+
+	return garageTomlPath, nil
+}
+
+func garageChildrenPmuxProcConfigs(env crypticnet.Env) ([]pmuxlib.ProcessConfig, error) {
+
+	var pmuxProcConfigs []pmuxlib.ProcessConfig
+
+	for _, alloc := range env.ThisDaemon().Storage.Allocations {
+
+		childConfPath, err := garageWriteChildConf(env, alloc)
+
+		if err != nil {
+			return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
+		}
+
+		pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
+			Name:        fmt.Sprintf("garage-%d", alloc.RPCPort),
+			Cmd:         "garage",
+			Args:        []string{"-c", childConfPath, "server"},
+			SigKillWait: 1 * time.Minute,
+		})
+	}
+
+	return pmuxProcConfigs, nil
+}
+
+func garageApplyLayoutDiffPmuxProcConfig(env crypticnet.Env) pmuxlib.ProcessConfig {
+	return pmuxlib.ProcessConfig{
+		Name:        "garage-apply-layout-diff",
+		Cmd:         "bash",
+		Args:        waitForGarageArgs(env, "bash", "garage-apply-layout-diff"),
+		NoRestartOn: []int{0},
+	}
+}
+
+func garageInitializeGlobalBucket(ctx context.Context, env crypticnet.Env) error {
+
+	var (
+		adminClient       = env.GarageAdminClient()
+		globalBucketCreds = env.Bootstrap.GarageGlobalBucketS3APICredentials
+	)
+
+	// first attempt to import the key
+	err := adminClient.Do(ctx, nil, "POST", "/v0/key/import", map[string]string{
+		"accessKeyId":     globalBucketCreds.ID,
+		"secretAccessKey": globalBucketCreds.Secret,
+		"name":            "shared-global-bucket-key",
+	})
+
+	if err != nil {
+		return fmt.Errorf("importing global bucket key into garage: %w", err)
+	}
+
+	// create global bucket
+	err = adminClient.Do(ctx, nil, "POST", "/v0/bucket", map[string]string{
+		"globalAlias": garage.GlobalBucket,
+	})
+
+	if err != nil {
+		return fmt.Errorf("creating global bucket: %w", err)
+	}
+
+	// retrieve newly created bucket's id
+	var getBucketRes struct {
+		ID string `json:"id"`
+	}
+
+	err = adminClient.Do(
+		ctx, &getBucketRes,
+		"GET", "/v0/bucket?globalAlias="+garage.GlobalBucket, nil,
+	)
+
+	if err != nil {
+		return fmt.Errorf("fetching global bucket id: %w", err)
+	}
+
+	// allow shared global bucket key to perform all operations
+	err = adminClient.Do(ctx, nil, "POST", "/v0/bucket/allow", map[string]interface{}{
+		"bucketId":    getBucketRes.ID,
+		"accessKeyId": globalBucketCreds.ID,
+		"permissions": map[string]bool{
+			"read":  true,
+			"write": true,
+		},
+	})
+
+	if err != nil {
+		return fmt.Errorf("granting permissions to shared global bucket key: %w", err)
+	}
+
+	return nil
+}
+
+func garageApplyLayout(ctx context.Context, env crypticnet.Env) error {
+
+	var (
+		adminClient = env.GarageAdminClient()
+		thisHost    = env.Bootstrap.ThisHost()
+		hostName    = thisHost.Name
+		ip          = thisHost.Nebula.IP
+		allocs      = env.ThisDaemon().Storage.Allocations
+	)
+
+	type peerLayout struct {
+		Capacity int      `json:"capacity"`
+		Zone     string   `json:"zone"`
+		Tags     []string `json:"tags"`
+	}
+
+	{
+		clusterLayout := map[string]peerLayout{}
+
+		for _, alloc := range allocs {
+
+			peer := garage.Peer{
+				IP:        ip,
+				RPCPort:   alloc.RPCPort,
+				S3APIPort: alloc.S3APIPort,
+			}
+
+			clusterLayout[peer.RPCPeerID()] = peerLayout{
+				Capacity: alloc.Capacity / 100,
+				Zone:     hostName,
+			}
+		}
+
+		err := adminClient.Do(ctx, nil, "POST", "/v0/layout", clusterLayout)
+		if err != nil {
+			return fmt.Errorf("staging layout changes: %w", err)
+		}
+	}
+
+	var clusterLayout struct {
+		Version           int                   `json:"version"`
+		StagedRoleChanges map[string]peerLayout `json:"stagedRoleChanges"`
+	}
+
+	if err := adminClient.Do(ctx, &clusterLayout, "GET", "/v0/layout", nil); err != nil {
+		return fmt.Errorf("retrieving staged layout change: %w", err)
+	}
+
+	if len(clusterLayout.StagedRoleChanges) == 0 {
+		return nil
+	}
+
+	applyClusterLayout := struct {
+		Version int `json:"version"`
+	}{
+		Version: clusterLayout.Version + 1,
+	}
+
+	err := adminClient.Do(ctx, nil, "POST", "/v0/layout/apply", applyClusterLayout)
+	if err != nil {
+		return fmt.Errorf("applying new layout (new version:%d): %w", applyClusterLayout.Version, err)
+	}
+
+	return nil
+}

go-workspace/src/cmd/entrypoint/sub_cmd.go

@@ -14,7 +14,7 @@ type subCmdCtx struct {
 	subCmd      subCmd   // the subCmd itself
 	args        []string // command-line arguments, excluding the subCmd itself.
 	subCmdNames []string // names of subCmds so far, including this one
-	env         *crypticnet.Env
+	env         crypticnet.Env
 }
 
 type subCmd struct {

go-workspace/src/cmd/garage-layout-diff/main.go

@@ -1,256 +0,0 @@
-package garage_layout_diff
-
-// This binary accepts the output of `garage layout show` into its stdout, and
-// it outputs a newline-delimited set of `garage layout $cmd` strings on
-// stdout. The layout commands which are output will, if run, bring the current
-// node's layout on the cluster up-to-date with what's in daemon.yml.
-
-import (
-	"bufio"
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"strconv"
-	"strings"
-
-	crypticnet "cryptic-net"
-	"cryptic-net/garage"
-)
-
-type clusterNode struct {
-	ID       string
-	Zone     string
-	Capacity int
-}
-
-type clusterNodes []clusterNode
-
-func (n clusterNodes) get(id string) (clusterNode, bool) {
-
-	var ok bool
-
-	for _, node := range n {
-
-		if len(node.ID) > len(id) {
-			ok = strings.HasPrefix(node.ID, id)
-		} else {
-			ok = strings.HasPrefix(id, node.ID)
-		}
-
-		if ok {
-			return node, true
-		}
-	}
-
-	return clusterNode{}, false
-}
-
-var currClusterLayoutVersionB = []byte("Current cluster layout version:")
-
-func readCurrNodes(r io.Reader) (clusterNodes, int, error) {
-
-	input, err := io.ReadAll(r)
-
-	if err != nil {
-		return nil, 0, fmt.Errorf("reading stdin: %w", err)
-	}
-
-	// NOTE I'm not sure if this check should be turned on or not. It simplifies
-	// things to turn it off and just say that no one should ever be manually
-	// messing with the layout, but on the other hand maybe someone might?
-	//
-	//if i := bytes.Index(input, []byte("==== STAGED ROLE CHANGES ====")); i >= 0 {
-	//	return nil, 0, errors.New("cluster layout has staged changes already, won't modify")
-	//}
-
-	/* The first section of input will always be something like this:
-
-	```
-	==== CURRENT CLUSTER LAYOUT ====
-	ID                 Tags  Zone              Capacity
-	AAA…                     ZZZ               1
-	BBB…                     ZZZ               1
-	CCC…                     ZZZ               1
-
-	Current cluster layout version: N
-	```
-
-	There may be more, depending on if the cluster already has changes staged,
-	but this will definitely be first. */
-
-	i := bytes.Index(input, currClusterLayoutVersionB)
-
-	if i < 0 {
-		return nil, 0, errors.New("no current cluster layout found in input")
-	}
-
-	input, tail := input[:i], input[i:]
-
-	var currNodes clusterNodes
-
-	for inputBuf := bufio.NewReader(bytes.NewBuffer(input)); ; {
-
-		line, err := inputBuf.ReadString('\n')
-
-		if errors.Is(err, io.EOF) {
-			break
-		} else if err != nil {
-			return nil, 0, fmt.Errorf("reading input line by line from buffer: %w", err)
-		}
-
-		fields := strings.Fields(line)
-
-		if len(fields) < 3 {
-			continue
-		}
-
-		id := fields[0]
-
-		// The ID will always be given ending in this fucked up ellipses
-		if trimmedID := strings.TrimSuffix(id, "…"); id == trimmedID {
-			continue
-		} else {
-			id = trimmedID
-		}
-
-		zone := fields[1]
-
-		capacity, err := strconv.Atoi(fields[2])
-
-		if err != nil {
-			return nil, 0, fmt.Errorf("parsing capacity %q: %w", fields[2], err)
-		}
-
-		currNodes = append(currNodes, clusterNode{
-			ID:       id,
-			Zone:     zone,
-			Capacity: capacity,
-		})
-	}
-
-	// parse current cluster version from tail
-	tail = bytes.TrimPrefix(tail, currClusterLayoutVersionB)
-
-	if i := bytes.Index(tail, []byte("\n")); i > 0 {
-		tail = tail[:i]
-	}
-
-	tail = bytes.TrimSpace(tail)
-
-	version, err := strconv.Atoi(string(tail))
-
-	if err != nil {
-		return nil, 0, fmt.Errorf("parsing version string from %q: %w", tail, err)
-	}
-
-	return currNodes, version, nil
-}
-
-func readExpNodes(env *crypticnet.Env) clusterNodes {
-
-	thisHost := env.Bootstrap.ThisHost()
-
-	var expNodes clusterNodes
-
-	for _, alloc := range env.ThisDaemon().Storage.Allocations {
-
-		peer := garage.Peer{
-			IP:        thisHost.Nebula.IP,
-			RPCPort:   alloc.RPCPort,
-			S3APIPort: alloc.S3APIPort,
-		}
-
-		id := peer.RPCPeerID()
-
-		expNodes = append(expNodes, clusterNode{
-			ID:       id,
-			Zone:     env.Bootstrap.HostName,
-			Capacity: alloc.Capacity / 100,
-		})
-	}
-
-	return expNodes
-}
-
-// NOTE: The id formatting for currNodes and expNodes is different; expNodes has
-// fully expanded ids, currNodes are abbreviated.
-
-func diff(currNodes, expNodes clusterNodes) []string {
-
-	var lines []string
-
-	for _, node := range currNodes {
-
-		if _, ok := expNodes.get(node.ID); !ok {
-			lines = append(
-				lines,
-				fmt.Sprintf("garage layout remove %s", node.ID),
-			)
-		}
-	}
-
-	for _, expNode := range expNodes {
-
-		currNode, ok := currNodes.get(expNode.ID)
-
-		currNode.ID = expNode.ID // so that equality checking works
-
-		if ok && currNode == expNode {
-			continue
-		}
-
-		lines = append(
-			lines,
-			fmt.Sprintf(
-				"garage layout assign %s -z %s -c %d",
-				expNode.ID,
-				expNode.Zone,
-				expNode.Capacity,
-			),
-		)
-	}
-
-	return lines
-}
-
-func Main() {
-
-	env, err := crypticnet.ReadEnv()
-
-	if err != nil {
-		panic(fmt.Errorf("reading environment: %w", err))
-	}
-
-	currNodes, currVersion, err := readCurrNodes(os.Stdin)
-
-	if err != nil {
-		panic(fmt.Errorf("reading current layout from stdin: %w", err))
-	}
-
-	thisCurrNodes := make(clusterNodes, 0, len(currNodes))
-
-	for _, node := range currNodes {
-
-		if env.Bootstrap.HostName != node.Zone {
-			continue
-		}
-
-		thisCurrNodes = append(thisCurrNodes, node)
-	}
-
-	expNodes := readExpNodes(env)
-
-	lines := diff(thisCurrNodes, expNodes)
-
-	if len(lines) == 0 {
-		return
-	}
-
-	for _, line := range lines {
-		fmt.Println(line)
-	}
-
-	fmt.Printf("garage layout apply --version %d\n", currVersion+1)
-}

go-workspace/src/cmd/garage-layout-diff/main_test.go

@@ -1,137 +0,0 @@
-package garage_layout_diff
-
-import (
-	"bytes"
-	"reflect"
-	"strconv"
-	"testing"
-)
-
-func TestReadCurrNodes(t *testing.T) {
-
-	expNodes := clusterNodes{
-		{
-			ID:       "AAA",
-			Zone:     "XXX",
-			Capacity: 1,
-		},
-		{
-			ID:       "BBB",
-			Zone:     "YYY",
-			Capacity: 2,
-		},
-		{
-			ID:       "CCC",
-			Zone:     "ZZZ",
-			Capacity: 3,
-		},
-	}
-
-	expVersion := 666
-
-	tests := []struct {
-		input      string
-		expNodes   clusterNodes
-		expVersion int
-	}{
-		{
-			input: `
-==== CURRENT CLUSTER LAYOUT ====
-ID                 Tags  Zone              Capacity
-AAA…                     XXX               1
-BBB…                     YYY               2
-CCC…                     ZZZ               3
-
-Current cluster layout version: 666
-`,
-			expNodes:   expNodes,
-			expVersion: expVersion,
-		},
-	}
-
-	for i, test := range tests {
-		t.Run(strconv.Itoa(i), func(t *testing.T) {
-
-			gotNodes, gotVersion, err := readCurrNodes(
-				bytes.NewBufferString(test.input),
-			)
-
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if gotVersion != test.expVersion {
-				t.Fatalf(
-					"expected version %d, got %d",
-					test.expVersion,
-					gotVersion,
-				)
-			}
-
-			if !reflect.DeepEqual(gotNodes, test.expNodes) {
-				t.Fatalf(
-					"expected nodes: %#v,\ngot nodes: %#v",
-					gotNodes,
-					test.expNodes,
-				)
-			}
-
-		})
-	}
-}
-
-func TestDiff(t *testing.T) {
-
-	currNodes := clusterNodes{
-		{
-			ID:       "1",
-			Zone:     "zone",
-			Capacity: 1,
-		},
-		{
-			ID:       "2",
-			Zone:     "zone",
-			Capacity: 1,
-		},
-		{
-			ID:       "3",
-			Zone:     "zone",
-			Capacity: 1,
-		},
-		{
-			ID:       "4",
-			Zone:     "zone",
-			Capacity: 1,
-		},
-	}
-
-	expNodes := clusterNodes{
-		{
-			ID:       "111",
-			Zone:     "zone",
-			Capacity: 1,
-		},
-		{
-			ID:       "222",
-			Zone:     "zone2",
-			Capacity: 1,
-		},
-		{
-			ID:       "333",
-			Zone:     "zone",
-			Capacity: 10,
-		},
-	}
-
-	expLines := []string{
-		`garage layout remove 4`,
-		`garage layout assign 222 -z zone2 -c 1`,
-		`garage layout assign 333 -z zone -c 10`,
-	}
-
-	gotLines := diff(currNodes, expNodes)
-
-	if !reflect.DeepEqual(gotLines, expLines) {
-		t.Fatalf("expected lines: %#v,\ngot lines: %#v", expLines, gotLines)
-	}
-}

go-workspace/src/cmd/garage-peer-keygen/main.go

@@ -1,65 +0,0 @@
-package garage_peer_keygen
-
-/*
-
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-!!                                                                            !!
-!!                                   DANGER                                   !!
-!!                                                                            !!
-!! This script will deterministically produce public/private keys given some  !!
-!! arbitrary input. This is NEVER what you want. It's only being used in      !!
-!! cryptic-net for a very specific purpose for which I think it's ok and is   !!
-!! very necessary, and people are probably _still_ going to yell at me.       !!
-!!                                                                            !!
-!!                                DONT USE THIS.                              !!
-!!                                                                            !!
-!! - Brian                                                                    !!
-!!                                                                            !!
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-
-*/
-
-import (
-	"encoding/hex"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"os"
-
-	"cryptic-net/garage"
-)
-
-func Main() {
-
-	ip := flag.String("ip", "", "Internal IP address of the node to generate a key for")
-	port := flag.Int("port", 0, "RPC port number for the garage instance to generate a key for")
-	outPriv := flag.String("out-priv", "", "The path to the private key which should be created, if given.")
-	outPub := flag.String("out-pub", "", "The path to the public key which should be created, if given.")
-	danger := flag.Bool("danger", false, "Set this flag to indicate you understand WHY this binary should NEVER be used (see source code).")
-	flag.Parse()
-
-	if len(*ip) == 0 || *port == 0 || !*danger {
-		panic("The arguments -ip, -port, and -danger are required")
-	}
-
-	peer := garage.Peer{
-		IP:      *ip,
-		RPCPort: *port,
-	}
-
-	pubKey, privKey := peer.RPCPeerKey()
-
-	fmt.Fprintln(os.Stdout, hex.EncodeToString(pubKey))
-
-	if *outPub != "" {
-		if err := ioutil.WriteFile(*outPub, pubKey, 0444); err != nil {
-			panic(fmt.Errorf("writing public key to %q: %w", *outPub, err))
-		}
-	}
-
-	if *outPriv != "" {
-		if err := ioutil.WriteFile(*outPriv, privKey, 0400); err != nil {
-			panic(fmt.Errorf("writing private key to %q: %w", *outPriv, err))
-		}
-	}
-}

go-workspace/src/env.go

@@ -3,13 +3,16 @@ package crypticnet
 import (
 	"context"
 	"cryptic-net/bootstrap"
+	"cryptic-net/garage"
 	"cryptic-net/yamlutil"
 	"errors"
 	"fmt"
 	"io/fs"
+	"net"
 	"os"
 	"os/signal"
 	"path/filepath"
+	"strconv"
 	"sync"
 	"syscall"
 
@@ -56,24 +59,24 @@ func getAppDirPath() string {
 // If bootstrapOptional is true then NewEnv will first check if a bootstrap file
 // can be found in the expected places, and if not then it will not populate
 // BootstrapFS or any other fields based on it.
-func NewEnv(bootstrapOptional bool) (*Env, error) {
+func NewEnv(bootstrapOptional bool) (Env, error) {
 
 	runtimeDirPath := filepath.Join(xdg.RuntimeDir, "cryptic-net")
 	appDirPath := getAppDirPath()
 
-	env := &Env{
+	env := Env{
 		AppDirPath:     appDirPath,
 		DaemonYmlPath:  filepath.Join(runtimeDirPath, "daemon.yml"),
 		RuntimeDirPath: runtimeDirPath,
 		DataDirPath:    filepath.Join(xdg.DataHome, "cryptic-net"),
 	}
 
-	return env, env.init(bootstrapOptional)
+	return env.init(bootstrapOptional)
 }
 
 // ReadEnv reads an Env from the process's environment variables, rather than
 // calculating like NewEnv does.
-func ReadEnv() (*Env, error) {
+func ReadEnv() (Env, error) {
 
 	var err error
 
@@ -91,7 +94,7 @@ func ReadEnv() (*Env, error) {
 		return val
 	}
 
-	env := &Env{
+	env := Env{
 		AppDirPath:     getAppDirPath(),
 		DaemonYmlPath:  readEnv(DaemonYmlPathEnvVar),
 		RuntimeDirPath: readEnv(RuntimeDirPathEnvVar),
@@ -99,35 +102,35 @@ func ReadEnv() (*Env, error) {
 	}
 
 	if err != nil {
-		return nil, err
+		return Env{}, err
 	}
 
-	return env, env.init(false)
+	return env.init(false)
 }
 
 // DataDirBootstrapPath returns the path to the bootstrap file within the user's
 // data dir. If the file does not exist there it will be found in the AppDirPath
 // by ReloadBootstrap.
-func (e *Env) DataDirBootstrapPath() string {
+func (e Env) DataDirBootstrapPath() string {
 	return filepath.Join(e.DataDirPath, "bootstrap.tgz")
 }
 
-// LoadBootstrap sets BootstrapPath to the given value, and loads BootstrapFS
-// and all derived fields based on that.
-func (e *Env) LoadBootstrap(path string) error {
+// LoadBootstrap loads a Bootstrap from the given path, and returns a copy of
+// the Env with that Bootstrap set along with the BootstrapPath (or an error).
+func (e Env) LoadBootstrap(path string) (Env, error) {
 
 	var err error
 
 	if e.Bootstrap, err = bootstrap.FromFile(path); err != nil {
-		return fmt.Errorf("parsing bootstrap.tgz at %q: %w", path, err)
+		return Env{}, fmt.Errorf("parsing bootstrap.tgz at %q: %w", path, err)
 	}
 
 	e.BootstrapPath = path
 
-	return nil
+	return e, nil
 }
 
-func (e *Env) initBootstrap(bootstrapOptional bool) error {
+func (e Env) initBootstrap(bootstrapOptional bool) (Env, error) {
 
 	exists := func(path string) (bool, error) {
 		if _, err := os.Stat(path); errors.Is(err, fs.ErrNotExist) {
@@ -145,7 +148,7 @@ func (e *Env) initBootstrap(bootstrapOptional bool) error {
 		bootstrapPath := e.DataDirBootstrapPath()
 
 		if exists, err := exists(bootstrapPath); err != nil {
-			return fmt.Errorf("determining if %q exists: %w", bootstrapPath, err)
+			return Env{}, fmt.Errorf("determining if %q exists: %w", bootstrapPath, err)
 
 		} else if exists {
 			return e.LoadBootstrap(bootstrapPath)
@@ -158,20 +161,20 @@ func (e *Env) initBootstrap(bootstrapOptional bool) error {
 		bootstrapPath := filepath.Join(e.AppDirPath, "share/bootstrap.tgz")
 
 		if exists, err := exists(bootstrapPath); err != nil {
-			return fmt.Errorf("determining if %q exists: %w", bootstrapPath, err)
+			return Env{}, fmt.Errorf("determining if %q exists: %w", bootstrapPath, err)
 
 		} else if !exists && !bootstrapOptional {
-			return fmt.Errorf("boostrap file not found at %q", bootstrapPath)
+			return Env{}, fmt.Errorf("boostrap file not found at %q", bootstrapPath)
 
 		} else if exists {
 			return e.LoadBootstrap(bootstrapPath)
 		}
 	}
 
-	return nil
+	return e, nil
 }
 
-func (e *Env) init(bootstrapOptional bool) error {
+func (e Env) init(bootstrapOptional bool) (Env, error) {
 
 	var cancel context.CancelFunc
 	e.Context, cancel = context.WithCancel(context.Background())
@@ -191,16 +194,12 @@ func (e *Env) init(bootstrapOptional bool) error {
 		os.Exit(1)
 	}()
 
-	if err := e.initBootstrap(bootstrapOptional); err != nil {
-		return fmt.Errorf("initializing bootstrap data: %w", err)
-	}
-
-	return nil
+	return e.initBootstrap(bootstrapOptional)
 }
 
 // ToMap returns the Env as a map of key/value strings. If this map is set into
 // a process's environment, then that process can read it back using ReadEnv.
-func (e *Env) ToMap() map[string]string {
+func (e Env) ToMap() map[string]string {
 	return map[string]string{
 		DaemonYmlPathEnvVar:  e.DaemonYmlPath,
 		BootstrapPathEnvVar:  e.BootstrapPath,
@@ -211,7 +210,7 @@ func (e *Env) ToMap() map[string]string {
 
 // ThisDaemon returns the DaemonYml (loaded from DaemonYmlPath) for the
 // currently running process.
-func (e *Env) ThisDaemon() DaemonYml {
+func (e Env) ThisDaemon() DaemonYml {
 	e.thisDaemonOnce.Do(func() {
 		if err := yamlutil.LoadYamlFile(&e.thisDaemon, e.DaemonYmlPath); err != nil {
 			panic(err)
@@ -221,6 +220,22 @@ func (e *Env) ThisDaemon() DaemonYml {
 }
 
 // BinPath returns the absolute path to a binary in the AppDir.
-func (e *Env) BinPath(name string) string {
+func (e Env) BinPath(name string) string {
 	return filepath.Join(e.AppDirPath, "bin", name)
 }
+
+// GarageAdminClient will return an AdminClient for a local garage instance, or
+// it will _panic_ if there is no local instance configured.
+func (e Env) GarageAdminClient() *garage.AdminClient {
+
+	thisHost := e.Bootstrap.ThisHost()
+	thisDaemon := e.ThisDaemon()
+
+	return garage.NewAdminClient(
+		net.JoinHostPort(
+			thisHost.Nebula.IP,
+			strconv.Itoa(thisDaemon.Storage.Allocations[0].AdminPort),
+		),
+		e.Bootstrap.GarageAdminToken,
+	)
+}

go-workspace/src/garage/admin_client.go

@@ -82,3 +82,38 @@ func (c *AdminClient) Do(
 
 	return nil
 }
+
+// Wait will block until the instance connected to can see at least
+// ReplicationFactor-1 other garage instances. If the context is canceled it
+// will return the context error.
+func (c *AdminClient) Wait(ctx context.Context) error {
+	for {
+
+		var clusterStatus struct {
+			KnownNodes map[string]struct {
+				IsUp bool `json:"is_up"`
+			} `json:"knownNodes"`
+		}
+
+		err := c.Do(ctx, &clusterStatus, "GET", "/v0/status", nil)
+
+		if ctxErr := ctx.Err(); ctxErr != nil {
+			return ctxErr
+
+		} else if err != nil {
+			continue
+		}
+
+		var numUp int
+
+		for _, knownNode := range clusterStatus.KnownNodes {
+			if knownNode.IsUp {
+				numUp++
+			}
+		}
+
+		if numUp >= ReplicationFactor-1 {
+			return nil
+		}
+	}
+}

go-workspace/src/garage/client.go

@@ -1,12 +1,22 @@
 package garage
 
 import (
+	"crypto/rand"
+	"encoding/hex"
 	"errors"
 
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 )
 
+func randStr(l int) string {
+	b := make([]byte, l)
+	if _, err := rand.Read(b); err != nil {
+		panic(err)
+	}
+	return hex.EncodeToString(b)
+}
+
 // IsKeyNotFound returns true if the given error is the result of a key not
 // being found in a bucket.
 func IsKeyNotFound(err error) bool {
@@ -24,6 +34,14 @@ type S3APICredentials struct {
 	Secret string `yaml:"secret"`
 }
 
+// NewS3APICredentials returns a new usable instance of S3APICredentials.
+func NewS3APICredentials() S3APICredentials {
+	return S3APICredentials{
+		ID:     randStr(8),
+		Secret: randStr(32),
+	}
+}
+
 // NewS3APIClient returns a minio client configured to use the given garage S3 API
 // endpoint.
 func NewS3APIClient(addr string, creds S3APICredentials) (S3APIClient, error) {

go-workspace/src/garage/garage.go

@@ -10,4 +10,8 @@ const (
 	// GlobalBucket is the name of the global garage bucket which is
 	// accessible to all hosts in the network.
 	GlobalBucket = "cryptic-net-global"
+
+	// ReplicationFactor indicates the replication factor set on the garage
+	// cluster. We currently only support a factor of 3.
+	ReplicationFactor = 3
 )

go-workspace/src/nebula/nebula.go

@@ -14,15 +14,6 @@ import (
 	"golang.org/x/crypto/curve25519"
 )
 
-// TODO this should one day not be hardcoded
-var ipCIDRMask = func() net.IPMask {
-	_, ipNet, err := net.ParseCIDR("10.10.0.0/16")
-	if err != nil {
-		panic(err)
-	}
-	return ipNet.Mask
-}()
-
 // HostCert contains the certificate and private key files which will need to
 // be present on a particular host. Each file is PEM encoded.
 type HostCert struct {
@@ -41,7 +32,7 @@ type CACert struct {
 // NewHostCert generates a new key/cert for a nebula host using the CA key
 // which will be found in the adminFS.
 func NewHostCert(
-	caCert CACert, hostName, hostIP string,
+	caCert CACert, hostName string, ip net.IP,
 ) (
 	HostCert, error,
 ) {
@@ -66,14 +57,9 @@ func NewHostCert(
 
 	expireAt := caCrt.Details.NotAfter.Add(-1 * time.Second)
 
-	ip := net.ParseIP(hostIP)
-	if ip == nil {
-		return HostCert{}, fmt.Errorf("invalid host ip %q", hostIP)
-	}
-
-	ipNet := &net.IPNet{
-		IP:   ip,
-		Mask: ipCIDRMask,
+	subnet := caCrt.Details.Subnets[0]
+	if !subnet.Contains(ip) {
+		return HostCert{}, fmt.Errorf("invalid ip %q, not contained by network subnet %q", ip, subnet)
 	}
 
 	var hostPub, hostKey []byte
@@ -89,7 +75,10 @@ func NewHostCert(
 	hostCrt := cert.NebulaCertificate{
 		Details: cert.NebulaCertificateDetails{
 			Name: hostName,
-			Ips:       []*net.IPNet{ipNet},
+			Ips: []*net.IPNet{{
+				IP:   ip,
+				Mask: subnet.Mask,
+			}},
 			NotBefore: time.Now(),
 			NotAfter:  expireAt,
 			PublicKey: hostPub,
@@ -122,7 +111,7 @@ func NewHostCert(
 
 // NewCACert generates a CACert. The domain should be the network's root domain,
 // and is included in the signing certificate's Name field.
-func NewCACert(domain string) (CACert, error) {
+func NewCACert(domain string, subnet *net.IPNet) (CACert, error) {
 
 	pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
 	if err != nil {
@@ -135,6 +124,7 @@ func NewCACert(domain string) (CACert, error) {
 	caCrt := cert.NebulaCertificate{
 		Details: cert.NebulaCertificateDetails{
 			Name:      fmt.Sprintf("%s cryptic-net root cert", domain),
+			Subnets:   []*net.IPNet{subnet},
 			NotBefore: now,
 			NotAfter:  expireAt,
 			PublicKey: pubKey,