WIP

The daemon entrypoint now starts the garage child processes directly,
without the extra step of indirection

dnsmasq/bin/dnsmasq-entrypoint

@@ -16,13 +16,15 @@ tmp="$(mktemp -d -t cryptic-net-dnsmasq-entrypoint-XXX)"
     thisHostName=$(tar xzf "$_BOOTSTRAP_PATH" --to-stdout ./hostname)
     thisHostIP=$(cat "$tmp"/hosts/"$thisHostName".yml | yq '.nebula.ip')
 
+    domain=$(tar xzf "$_BOOTSTRAP_PATH" --to-stdout ./admin/creation-params.yml | yq '.domain')
+
     echo "listen-address=$thisHostIP" >> "$conf_path"
 
     ls -1 "$tmp"/hosts | while read hostYml; do
 
         hostName=$(echo "$hostYml" | cut -d. -f1)
         hostIP=$(cat "$tmp"/hosts/"$hostYml" | yq '.nebula.ip')
-        echo "address=/${hostName}.hosts.cryptic.io/$hostIP" >> "$conf_path"
+        echo "address=/${hostName}.hosts.$domain/$hostIP" >> "$conf_path"
 
     done
 )

docs/dev/daemon-process-tree.plantuml

@@ -17,6 +17,7 @@ state AppDir {
         entrypoint : * Merge given and default daemon.yml files
         entrypoint : * Copy bootstrap.tgz into $_DATA_DIR_PATH, if it's not there
         entrypoint : * Merge daemon.yml config into bootstrap.tgz
+        entrypoint : * Create $_RUNTIME_DIR_PATH/garage-N.toml\n  (one per storage allocation)
         entrypoint : * Run child processes
     }
 
@@ -41,20 +42,17 @@ state AppDir {
     entrypoint --> nebulaEntrypoint : child
     nebulaEntrypoint --> nebula : exec
 
-    state "./bin/cryptic-net-main garage-entrypoint" as garageEntrypoint {
+    state "Garage processes (only if any storage allocs are defined)" as garageChildren {
-        garageEntrypoint : * Create $_RUNTIME_DIR_PATH/garage-N.toml\n  (one per storage allocation)
-        garageEntrypoint : * Run child processes
-    }
 
         state "./bin/garage -c $_RUNTIME_DIR_PATH/garage-N.toml server" as garage
         state "./bin/garage-apply-layout-diff" as garageApplyLayoutDiff {
             garageApplyLayoutDiff : * Runs once then exits
             garageApplyLayoutDiff : * Updates cluster topo
         }
+    }
 
-    entrypoint --> garageEntrypoint : child (only if >1 storage allocation defined in daemon.yml)
+    entrypoint --> garage : child (one per storage allocation)
-    garageEntrypoint --> garage : child (one per storage allocation)
+    entrypoint --> garageApplyLayoutDiff : child
-    garageEntrypoint --> garageApplyLayoutDiff : child
 
     state "./bin/cryptic-net-main update-global-bucket" as updateGlobalBucket {
         updateGlobalBucket : * Runs once then exits

go-workspace/src/admin/admin.go

@@ -14,6 +14,8 @@ import (
 )
 
 const (
+	creationParamsPath = "admin/creation-params.yml"
+
 	nebulaCertsCACertPath = "nebula/certs/ca.crt"
 	nebulaCertsCAKeyPath  = "nebula/certs/ca.key"
 
@@ -22,8 +24,17 @@ const (
 	garageRPCSecretPath          = "garage/rpc-secret.txt"
 )
 
+// CreationParams are general parameters used when creating a new network. These
+// are available to all hosts within the network via their bootstrap files.
+type CreationParams struct {
+	ID     string `yaml:"id"`
+	Domain string `yaml:"domain"`
+}
+
 // Admin is used for accessing all information contained within an admin.tgz.
 type Admin struct {
+	CreationParams CreationParams
+
 	NebulaCACert nebula.CACert
 
 	GarageRPCSecret                    string
@@ -41,6 +52,7 @@ func FromFS(adminFS fs.FS) (Admin, error) {
 		into interface{}
 		path string
 	}{
+		{&a.CreationParams, creationParamsPath},
 		{&a.GarageGlobalBucketS3APICredentials, garageGlobalBucketKeyYmlPath},
 		{&a.GarageAdminBucketS3APICredentials, garageAdminBucketKeyYmlPath},
 	}
@@ -91,6 +103,7 @@ func (a Admin) WriteTo(into io.Writer) error {
 		value interface{}
 		path  string
 	}{
+		{a.CreationParams, creationParamsPath},
 		{a.GarageGlobalBucketS3APICredentials, garageGlobalBucketKeyYmlPath},
 		{a.GarageAdminBucketS3APICredentials, garageAdminBucketKeyYmlPath},
 	}

go-workspace/src/bootstrap/bootstrap.go

@@ -3,6 +3,7 @@
 package bootstrap
 
 import (
+	"cryptic-net/admin"
 	"cryptic-net/garage"
 	"cryptic-net/nebula"
 	"cryptic-net/tarutil"
@@ -20,12 +21,15 @@ import (
 
 // Paths within the bootstrap FS which for general data.
 const (
+	adminCreationParamsPath = "admin/creation-params.yml"
 	hostNamePath            = "hostname"
 )
 
 // Bootstrap is used for accessing all information contained within a
 // bootstrap.tgz file.
 type Bootstrap struct {
+	AdminCreationParams admin.CreationParams
+
 	Hosts    map[string]Host
 	HostName string
 
@@ -48,12 +52,18 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
 		return Bootstrap{}, fmt.Errorf("loading hosts info from fs: %w", err)
 	}
 
-	if err = yamlutil.LoadYamlFSFile(
+	filesToLoadAsYAML := []struct {
-		&b.GarageGlobalBucketS3APICredentials,
+		into interface{}
-		bootstrapFS,
+		path string
-		garageGlobalBucketKeyYmlPath,
+	}{
-	); err != nil {
+		{&b.AdminCreationParams, adminCreationParamsPath},
-		return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", garageGlobalBucketKeyYmlPath, err)
+		{&b.GarageGlobalBucketS3APICredentials, garageGlobalBucketKeyYmlPath},
+	}
+
+	for _, f := range filesToLoadAsYAML {
+		if err := yamlutil.LoadYamlFSFile(f.into, bootstrapFS, f.path); err != nil {
+			return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", f.path, err)
+		}
 	}
 
 	filesToLoadAsString := []struct {
@@ -106,6 +116,36 @@ func (b Bootstrap) WriteTo(into io.Writer) error {
 
 	w := tarutil.NewTGZWriter(into)
 
+	for _, host := range b.Hosts {
+
+		hostB, err := yaml.Marshal(host)
+		if err != nil {
+			return fmt.Errorf("yaml encoding host %#v: %w", host, err)
+		}
+
+		path := filepath.Join(hostsDirPath, host.Name+".yml")
+
+		w.WriteFileBytes(path, hostB)
+	}
+
+	filesToWriteAsYAML := []struct {
+		value interface{}
+		path  string
+	}{
+		{b.AdminCreationParams, adminCreationParamsPath},
+		{b.GarageGlobalBucketS3APICredentials, garageGlobalBucketKeyYmlPath},
+	}
+
+	for _, f := range filesToWriteAsYAML {
+
+		b, err := yaml.Marshal(f.value)
+		if err != nil {
+			return fmt.Errorf("yaml encoding data for %q: %w", f.path, err)
+		}
+
+		w.WriteFileBytes(f.path, b)
+	}
+
 	filesToWriteAsString := []struct {
 		value string
 		path  string
@@ -121,25 +161,6 @@ func (b Bootstrap) WriteTo(into io.Writer) error {
 		w.WriteFileBytes(f.path, []byte(f.value))
 	}
 
-	garageGlobalBucketKeyB, err := yaml.Marshal(b.GarageGlobalBucketS3APICredentials)
-	if err != nil {
-		return fmt.Errorf("yaml encoding garage global bucket creds: %w", err)
-	}
-
-	w.WriteFileBytes(garageGlobalBucketKeyYmlPath, garageGlobalBucketKeyB)
-
-	for _, host := range b.Hosts {
-
-		hostB, err := yaml.Marshal(host)
-		if err != nil {
-			return fmt.Errorf("yaml encoding host %#v: %w", host, err)
-		}
-
-		path := filepath.Join(hostsDirPath, host.Name+".yml")
-
-		w.WriteFileBytes(path, hostB)
-	}
-
 	return w.Close()
 }
 

go-workspace/src/cmd/cryptic-net-main/main.go

@@ -16,7 +16,6 @@ package main
 
 import (
 	"cryptic-net/cmd/entrypoint"
-	garage_entrypoint "cryptic-net/cmd/garage-entrypoint"
 	garage_layout_diff "cryptic-net/cmd/garage-layout-diff"
 	garage_peer_keygen "cryptic-net/cmd/garage-peer-keygen"
 	nebula_entrypoint "cryptic-net/cmd/nebula-entrypoint"
@@ -32,7 +31,6 @@ type mainFn struct {
 
 var mainFns = []mainFn{
 	{"entrypoint", entrypoint.Main},
-	{"garage-entrypoint", garage_entrypoint.Main},
 	{"garage-layout-diff", garage_layout_diff.Main},
 	{"garage-peer-keygen", garage_peer_keygen.Main},
 	{"nebula-entrypoint", nebula_entrypoint.Main},

go-workspace/src/cmd/entrypoint/admin.go

@@ -1,14 +1,29 @@
 package entrypoint
 
 import (
+	"context"
 	"cryptic-net/admin"
 	"cryptic-net/bootstrap"
 	"cryptic-net/nebula"
+	"crypto/rand"
+	"encoding/hex"
 	"errors"
 	"fmt"
+	"net"
 	"os"
+	"strings"
+
+	"github.com/cryptic-io/pmux/pmuxlib"
 )
 
+func randStr(l int) string {
+	b := make([]byte, l)
+	if _, err := rand.Read(b); err != nil {
+		panic(err)
+	}
+	return hex.EncodeToString(b)
+}
+
 func readAdmin(path string) (admin.Admin, error) {
 
 	if path == "-" {
@@ -30,6 +45,155 @@ func readAdmin(path string) (admin.Admin, error) {
 	return admin.FromReader(f)
 }
 
+var subCmdAdminCreateNetwork = subCmd{
+	name:  "create-network",
+	descr: "Creates a new cryptic-net network, outputting the resulting admin.tgz to stdout",
+	do: func(subCmdCtx subCmdCtx) error {
+
+		flags := subCmdCtx.flagSet(false)
+
+		daemonYmlPath := flags.StringP(
+			"config-path", "c", "",
+			"Optional path to a daemon.yml file to load configuration from.",
+		)
+
+		dumpConfig := flags.Bool(
+			"dump-config", false,
+			"Write the default configuration file to stdout and exit.",
+		)
+
+		domain := flags.StringP(
+			"domain", "d", "",
+			"Domain name that should be used as the root domain in the network.",
+		)
+
+		subnetStr := flags.StringP(
+			"subnet", "s", "",
+			"CIDR which denotes the subnet that IPs hosts on the network can be assigned.",
+		)
+
+		if err := flags.Parse(subCmdCtx.args); err != nil {
+			return fmt.Errorf("parsing flags: %w", err)
+		}
+
+		env := subCmdCtx.env
+
+		if *dumpConfig {
+			return writeBuiltinDaemonYml(env, os.Stdout)
+		}
+
+		if *domain == "" || *subnetStr == "" {
+			return errors.New("--domain and --subnet are required")
+		}
+
+		*domain = strings.TrimRight(strings.TrimLeft(*domain, "."), ".")
+
+		ip, subnet, err := net.ParseCIDR(*subnetStr)
+		if err != nil {
+			return fmt.Errorf("parsing %q as a CIDR: %w", *subnetStr, err)
+		}
+
+		hostName := "genesis"
+
+		adminCreationParams := admin.CreationParams{
+			ID:     randStr(32),
+			Domain: *domain,
+		}
+
+		garageRPCSecret := randStr(32)
+
+		{
+			runtimeDirPath := env.RuntimeDirPath
+
+			fmt.Fprintf(os.Stderr, "will use runtime directory %q for temporary state\n", runtimeDirPath)
+
+			if err := os.MkdirAll(runtimeDirPath, 0700); err != nil {
+				return fmt.Errorf("creating directory %q: %w", runtimeDirPath, err)
+			}
+
+			defer func() {
+				fmt.Fprintf(os.Stderr, "cleaning up runtime directory %q\n", runtimeDirPath)
+				if err := os.RemoveAll(runtimeDirPath); err != nil {
+					fmt.Fprintf(os.Stderr, "error removing temporary directory %q: %v", runtimeDirPath, err)
+				}
+			}()
+		}
+
+		if err := writeMergedDaemonYml(env, *daemonYmlPath); err != nil {
+			return fmt.Errorf("merging and writing daemon.yml file: %w", err)
+		}
+
+		daemon := env.ThisDaemon()
+
+		if len(daemon.Storage.Allocations) < 3 {
+			return fmt.Errorf("daemon.yml with at least 3 allocations was not provided")
+		}
+
+		nebulaCACert, err := nebula.NewCACert(*domain, subnet)
+		if err != nil {
+			return fmt.Errorf("creating nebula CA cert: %w", err)
+		}
+
+		nebulaHostCert, err := nebula.NewHostCert(nebulaCACert, hostName, ip)
+		if err != nil {
+			return fmt.Errorf("creating nebula cert for host: %w", err)
+		}
+
+		host := bootstrap.Host{
+			Name: hostName,
+			Nebula: bootstrap.NebulaHost{
+				IP: ip.String(),
+			},
+		}
+
+		env.Bootstrap = bootstrap.Bootstrap{
+			AdminCreationParams: adminCreationParams,
+			Hosts: map[string]bootstrap.Host{
+				hostName: host,
+			},
+			HostName:        hostName,
+			NebulaHostCert:  nebulaHostCert,
+			GarageRPCSecret: garageRPCSecret,
+		}
+
+		// this will also write the bootstrap file
+		if err := mergeDaemonIntoBootstrap(env); err != nil {
+			return fmt.Errorf("merging daemon.yml into bootstrap data: %w", err)
+		}
+
+		for key, val := range env.ToMap() {
+			if err := os.Setenv(key, val); err != nil {
+				return fmt.Errorf("failed to set %q to %q: %w", key, val, err)
+			}
+		}
+
+		garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env)
+		if err != nil {
+			return fmt.Errorf("generating garage children configs: %w", err)
+		}
+
+		pmuxConfig := pmuxlib.Config{
+			Processes: append(
+				[]pmuxlib.ProcessConfig{
+					nebulaEntrypointPmuxProcConfig(),
+					garageApplyLayoutDiffPmuxProcConfig(env),
+				},
+				garageChildrenPmuxProcConfigs...,
+			),
+		}
+
+		ctx, cancel := context.WithCancel(env.Context)
+		defer cancel()
+		pmuxDoneCh := make(chan struct{})
+
+		go func() {
+			pmuxlib.Run(ctx, pmuxConfig)
+			close(pmuxDoneCh)
+		}()
+
+	},
+}
+
 var subCmdAdminMakeBootstrap = subCmd{
 	name:      "make-bootstrap",
 	descr:     "Creates a new bootstrap.tgz file for a particular host and writes it to stdout",
@@ -88,6 +252,8 @@ var subCmdAdminMakeBootstrap = subCmd{
 		}
 
 		newBootstrap := bootstrap.Bootstrap{
+			AdminCreationParams: adm.CreationParams,
+
 			Hosts:    hosts,
 			HostName: *name,
 

go-workspace/src/cmd/entrypoint/daemon.go

@@ -5,23 +5,15 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io"
-	"io/ioutil"
-	"net"
 	"os"
-	"path/filepath"
-	"strconv"
 	"sync"
 	"time"
 
 	crypticnet "cryptic-net"
 	"cryptic-net/bootstrap"
 	"cryptic-net/garage"
-	"cryptic-net/yamlutil"
 
 	"github.com/cryptic-io/pmux/pmuxlib"
-	"github.com/imdario/mergo"
-	"gopkg.in/yaml.v3"
 )
 
 // The daemon sub-command deals with starting an actual cryptic-net daemon
@@ -46,70 +38,6 @@ import (
 //
 // * (On exit) cleans up the runtime directory.
 
-func writeDaemonYml(userDaemonYmlPath, builtinDaemonYmlPath, runtimeDirPath string) error {
-
-	var fullDaemonYml map[string]interface{}
-
-	if err := yamlutil.LoadYamlFile(&fullDaemonYml, builtinDaemonYmlPath); err != nil {
-		return fmt.Errorf("parsing builtin daemon.yml file: %w", err)
-	}
-
-	if userDaemonYmlPath != "" {
-
-		var daemonYml map[string]interface{}
-		if err := yamlutil.LoadYamlFile(&daemonYml, userDaemonYmlPath); err != nil {
-			return fmt.Errorf("parsing %q: %w", userDaemonYmlPath, err)
-		}
-
-		err := mergo.Merge(&fullDaemonYml, daemonYml, mergo.WithOverride)
-		if err != nil {
-			return fmt.Errorf("merging contents of file %q: %w", userDaemonYmlPath, err)
-		}
-	}
-
-	fullDaemonYmlB, err := yaml.Marshal(fullDaemonYml)
-
-	if err != nil {
-		return fmt.Errorf("yaml marshaling daemon config: %w", err)
-	}
-
-	daemonYmlPath := filepath.Join(runtimeDirPath, "daemon.yml")
-
-	if err := ioutil.WriteFile(daemonYmlPath, fullDaemonYmlB, 0400); err != nil {
-		return fmt.Errorf("writing daemon.yml file to %q: %w", daemonYmlPath, err)
-	}
-
-	return nil
-}
-
-func copyBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
-
-	path := env.DataDirBootstrapPath()
-	dirPath := filepath.Dir(path)
-
-	if err := os.MkdirAll(dirPath, 0700); err != nil {
-		return fmt.Errorf("creating directory %q: %w", dirPath, err)
-	}
-
-	f, err := os.Create(path)
-	if err != nil {
-		return fmt.Errorf("creating file %q: %w", path, err)
-	}
-
-	_, err = io.Copy(f, r)
-	f.Close()
-
-	if err != nil {
-		return fmt.Errorf("copying bootstrap file to %q: %w", path, err)
-	}
-
-	if err := env.LoadBootstrap(path); err != nil {
-		return fmt.Errorf("loading bootstrap from %q: %w", path, err)
-	}
-
-	return nil
-}
-
 // creates a new bootstrap file using available information from the network. If
 // the new bootstrap file is different than the existing one, the existing one
 // is overwritten, ReloadBootstrap is called on env, true is returned.
@@ -156,65 +84,31 @@ func runDaemonPmuxOnce(env *crypticnet.Env, s3Client garage.S3APIClient) error {
 	fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP)
 
 	pmuxProcConfigs := []pmuxlib.ProcessConfig{
-		{
+		nebulaEntrypointPmuxProcConfig(),
-			Name: "nebula",
-			Cmd:  "cryptic-net-main",
-			Args: []string{
-				"nebula-entrypoint",
-			},
-		},
 		{
 			Name: "dnsmasq",
 			Cmd:  "bash",
-			Args: []string{
+			Args: waitForNebulaArgs(env, "dnsmasq-entrypoint"),
-				"wait-for-ip",
-				thisHost.Nebula.IP,
-				"dnsmasq-entrypoint",
-			},
 		},
 	}
 
-	{
+	if len(thisDaemon.Storage.Allocations) > 0 {
-		var args []string
 
-		if allocs := thisDaemon.Storage.Allocations; len(allocs) > 0 {
+		garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env)
-			for _, alloc := range allocs {
+		if err != nil {
-				args = append(
+			return fmt.Errorf("generating garage children configs: %w", err)
-					args,
-					"wait-for",
-					net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
-					"--",
-				)
-			}
-		} else {
-			args = []string{
-				"wait-for-ip",
-				thisHost.Nebula.IP,
 		}
+
+		pmuxProcConfigs = append(pmuxProcConfigs, garageChildrenPmuxProcConfigs...)
+		pmuxProcConfigs = append(pmuxProcConfigs, garageApplyLayoutDiffPmuxProcConfig(env))
 	}
 
 	pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
 		Name:        "update-global-bucket",
 		Cmd:         "bash",
-			Args:        append(args, "update-global-bucket"),
+		Args:        waitForGarageArgs(env, "update-global-bucket"),
 		NoRestartOn: []int{0},
 	})
-	}
-
-	if len(thisDaemon.Storage.Allocations) > 0 {
-		pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
-			Name: "garage",
-			Cmd:  "bash",
-			Args: []string{
-				"wait-for-ip",
-				thisHost.Nebula.IP,
-				"cryptic-net-main", "garage-entrypoint",
-			},
-
-			// garage can take a while to clean up
-			SigKillWait: (1 * time.Minute) + (10 * time.Second),
-		})
-	}
 
 	pmuxConfig := pmuxlib.Config{Processes: pmuxProcConfigs}
 
@@ -284,28 +178,8 @@ var subCmdDaemon = subCmd{
 
 		env := subCmdCtx.env
 
-		s3Client, err := env.Bootstrap.GlobalBucketS3APIClient()
-		if err != nil {
-			return fmt.Errorf("creating client for global bucket: %w", err)
-		}
-
-		appDirPath := env.AppDirPath
-
-		builtinDaemonYmlPath := filepath.Join(appDirPath, "etc", "daemon.yml")
-
 		if *dumpConfig {
-
+			return writeBuiltinDaemonYml(env, os.Stdout)
-			builtinDaemonYml, err := os.ReadFile(builtinDaemonYmlPath)
-
-			if err != nil {
-				return fmt.Errorf("reading default daemon.yml at %q: %w", builtinDaemonYmlPath, err)
-			}
-
-			if _, err := os.Stdout.Write(builtinDaemonYml); err != nil {
-				return fmt.Errorf("writing default daemon.yml to stdout: %w", err)
-			}
-
-			return nil
 		}
 
 		runtimeDirPath := env.RuntimeDirPath
@@ -360,48 +234,17 @@ var subCmdDaemon = subCmd{
 			}
 		}
 
-		if err := writeDaemonYml(*daemonYmlPath, builtinDaemonYmlPath, runtimeDirPath); err != nil {
+		if err := writeMergedDaemonYml(env, *daemonYmlPath); err != nil {
-			return fmt.Errorf("generating daemon.yml file: %w", err)
+			return fmt.Errorf("merging and writing daemon.yml file: %w", err)
 		}
 
-		{
 		// we update this Host's data using whatever configuration has been
 		// provided by daemon.yml. This way the daemon has the most
 		// up-to-date possible bootstrap. This updated bootstrap will later
 		// get updated in garage using update-global-bucket, so other hosts
 		// will see it as well.
-
+		if err := mergeDaemonIntoBootstrap(env); err != nil {
-			// ThisDaemon can only be called after writeDaemonYml.
+			return fmt.Errorf("merging daemon.yml into bootstrap data: %w", err)
-			daemon := env.ThisDaemon()
-			host := env.Bootstrap.ThisHost()
-
-			host.Nebula.PublicAddr = daemon.VPN.PublicAddr
-
-			host.Garage = nil
-
-			if allocs := daemon.Storage.Allocations; len(allocs) > 0 {
-
-				host.Garage = new(bootstrap.GarageHost)
-
-				for _, alloc := range allocs {
-					host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
-						RPCPort:   alloc.RPCPort,
-						S3APIPort: alloc.S3APIPort,
-						WebPort:   alloc.WebPort,
-					})
-				}
-			}
-
-			env.Bootstrap.Hosts[host.Name] = host
-
-			buf := new(bytes.Buffer)
-			if err := env.Bootstrap.WithHosts(env.Bootstrap.Hosts).WriteTo(buf); err != nil {
-				return fmt.Errorf("writing new bootstrap file to buffer: %w", err)
-			}
-
-			if err := copyBootstrapToDataDir(env, buf); err != nil {
-				return fmt.Errorf("copying new bootstrap file to data dir: %w", err)
-			}
 		}
 
 		for key, val := range env.ToMap() {
@@ -412,6 +255,14 @@ var subCmdDaemon = subCmd{
 
 		for {
 
+			// create s3Client anew on every loop, in case the topology has
+			// changed and we should be connecting to a different garage
+			// endpoint.
+			s3Client, err := env.Bootstrap.GlobalBucketS3APIClient()
+			if err != nil {
+				return fmt.Errorf("creating client for global bucket: %w", err)
+			}
+
 			if err := runDaemonPmuxOnce(env, s3Client); errors.Is(err, context.Canceled) {
 				return nil
 

go-workspace/src/cmd/entrypoint/daemon_util.go

@@ -0,0 +1,205 @@
+package entrypoint
+
+import (
+	"bytes"
+	crypticnet "cryptic-net"
+	"cryptic-net/bootstrap"
+	"cryptic-net/garage"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/cryptic-io/pmux/pmuxlib"
+)
+
+func copyBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
+
+	path := env.DataDirBootstrapPath()
+	dirPath := filepath.Dir(path)
+
+	if err := os.MkdirAll(dirPath, 0700); err != nil {
+		return fmt.Errorf("creating directory %q: %w", dirPath, err)
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("creating file %q: %w", path, err)
+	}
+
+	_, err = io.Copy(f, r)
+	f.Close()
+
+	if err != nil {
+		return fmt.Errorf("copying bootstrap file to %q: %w", path, err)
+	}
+
+	if err := env.LoadBootstrap(path); err != nil {
+		return fmt.Errorf("loading bootstrap from %q: %w", path, err)
+	}
+
+	return nil
+}
+
+func mergeDaemonIntoBootstrap(env *crypticnet.Env) error {
+	daemon := env.ThisDaemon()
+	host := env.Bootstrap.ThisHost()
+
+	host.Nebula.PublicAddr = daemon.VPN.PublicAddr
+
+	host.Garage = nil
+
+	if allocs := daemon.Storage.Allocations; len(allocs) > 0 {
+
+		host.Garage = new(bootstrap.GarageHost)
+
+		for _, alloc := range allocs {
+			host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
+				RPCPort:   alloc.RPCPort,
+				S3APIPort: alloc.S3APIPort,
+				WebPort:   alloc.WebPort,
+			})
+		}
+	}
+
+	env.Bootstrap.Hosts[host.Name] = host
+
+	buf := new(bytes.Buffer)
+	if err := env.Bootstrap.WithHosts(env.Bootstrap.Hosts).WriteTo(buf); err != nil {
+		return fmt.Errorf("writing new bootstrap file to buffer: %w", err)
+	}
+
+	if err := copyBootstrapToDataDir(env, buf); err != nil {
+		return fmt.Errorf("copying new bootstrap file to data dir: %w", err)
+	}
+
+	return nil
+}
+
+func waitForNebulaArgs(env *crypticnet.Env, args ...string) []string {
+	thisHost := env.Bootstrap.ThisHost()
+	return append([]string{"wait-for-ip", thisHost.Nebula.IP}, args...)
+}
+
+func waitForGarageArgs(env *crypticnet.Env, args ...string) []string {
+
+	thisHost := env.Bootstrap.ThisHost()
+	allocs := env.ThisDaemon().Storage.Allocations
+
+	if len(allocs) == 0 {
+		return waitForNebulaArgs(env, args...)
+	}
+
+	var preArgs []string
+
+	for _, alloc := range allocs {
+		preArgs = append(
+			preArgs,
+			"wait-for",
+			net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
+			"--",
+		)
+	}
+
+	return append(preArgs, args...)
+}
+
+func nebulaEntrypointPmuxProcConfig() pmuxlib.ProcessConfig {
+	return pmuxlib.ProcessConfig{
+		Name: "nebula",
+		Cmd:  "cryptic-net-main",
+		Args: []string{
+			"nebula-entrypoint",
+		},
+	}
+}
+
+func garageWriteChildConf(
+	env *crypticnet.Env,
+	alloc crypticnet.DaemonYmlStorageAllocation,
+) (
+	string, error,
+) {
+
+	if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
+		return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
+	}
+
+	thisHost := env.Bootstrap.ThisHost()
+
+	peer := garage.Peer{
+		IP:        thisHost.Nebula.IP,
+		RPCPort:   alloc.RPCPort,
+		S3APIPort: alloc.S3APIPort,
+	}
+
+	pubKey, privKey := peer.RPCPeerKey()
+
+	nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
+	nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
+
+	if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
+		return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
+
+	} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
+		return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
+	}
+
+	garageTomlPath := filepath.Join(
+		env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
+	)
+
+	err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
+		MetaPath: alloc.MetaPath,
+		DataPath: alloc.DataPath,
+
+		RPCSecret: env.Bootstrap.GarageRPCSecret,
+
+		RPCAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
+		APIAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
+		WebAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.WebPort)),
+
+		BootstrapPeers: env.Bootstrap.GarageRPCPeerAddrs(),
+	})
+
+	if err != nil {
+		return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
+	}
+
+	return garageTomlPath, nil
+}
+
+func garageChildrenPmuxProcConfigs(env *crypticnet.Env) ([]pmuxlib.ProcessConfig, error) {
+
+	var pmuxProcConfigs []pmuxlib.ProcessConfig
+
+	for _, alloc := range env.ThisDaemon().Storage.Allocations {
+
+		childConfPath, err := garageWriteChildConf(env, alloc)
+
+		if err != nil {
+			return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
+		}
+
+		pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
+			Name:        fmt.Sprintf("garage-%d", alloc.RPCPort),
+			Cmd:         "garage",
+			Args:        []string{"-c", childConfPath, "server"},
+			SigKillWait: 1 * time.Minute,
+		})
+	}
+
+	return pmuxProcConfigs, nil
+}
+
+func garageApplyLayoutDiffPmuxProcConfig(env *crypticnet.Env) pmuxlib.ProcessConfig {
+	return pmuxlib.ProcessConfig{
+		Name:        "garage-apply-layout-diff",
+		Cmd:         "bash",
+		Args:        waitForGarageArgs(env, "bash", "garage-apply-layout-diff"),
+		NoRestartOn: []int{0},
+	}
+}

go-workspace/src/cmd/entrypoint/daemon_yml.go

@@ -0,0 +1,72 @@
+package entrypoint
+
+import (
+	crypticnet "cryptic-net"
+	"cryptic-net/yamlutil"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/imdario/mergo"
+	"gopkg.in/yaml.v3"
+)
+
+func builtinDaemonYmlPath(env *crypticnet.Env) string {
+	return filepath.Join(env.AppDirPath, "etc", "daemon.yml")
+}
+
+func writeBuiltinDaemonYml(env *crypticnet.Env, w io.Writer) error {
+
+	builtinDaemonYmlPath := builtinDaemonYmlPath(env)
+
+	builtinDaemonYml, err := os.ReadFile(builtinDaemonYmlPath)
+	if err != nil {
+		return fmt.Errorf("reading default daemon.yml at %q: %w", builtinDaemonYmlPath, err)
+	}
+
+	if _, err := w.Write(builtinDaemonYml); err != nil {
+		return fmt.Errorf("writing default daemon.yml: %w", err)
+	}
+
+	return nil
+}
+
+func writeMergedDaemonYml(env *crypticnet.Env, userDaemonYmlPath string) error {
+
+	builtinDaemonYmlPath := builtinDaemonYmlPath(env)
+
+	var fullDaemonYml map[string]interface{}
+
+	if err := yamlutil.LoadYamlFile(&fullDaemonYml, builtinDaemonYmlPath); err != nil {
+		return fmt.Errorf("parsing builtin daemon.yml file: %w", err)
+	}
+
+	if userDaemonYmlPath != "" {
+
+		var daemonYml map[string]interface{}
+		if err := yamlutil.LoadYamlFile(&daemonYml, userDaemonYmlPath); err != nil {
+			return fmt.Errorf("parsing %q: %w", userDaemonYmlPath, err)
+		}
+
+		err := mergo.Merge(&fullDaemonYml, daemonYml, mergo.WithOverride)
+		if err != nil {
+			return fmt.Errorf("merging contents of file %q: %w", userDaemonYmlPath, err)
+		}
+	}
+
+	fullDaemonYmlB, err := yaml.Marshal(fullDaemonYml)
+
+	if err != nil {
+		return fmt.Errorf("yaml marshaling daemon config: %w", err)
+	}
+
+	daemonYmlPath := filepath.Join(env.RuntimeDirPath, "daemon.yml")
+
+	if err := ioutil.WriteFile(daemonYmlPath, fullDaemonYmlB, 0400); err != nil {
+		return fmt.Errorf("writing daemon.yml file to %q: %w", daemonYmlPath, err)
+	}
+
+	return nil
+}

go-workspace/src/cmd/garage-entrypoint/main.go

@@ -1,129 +0,0 @@
-package garage_entrypoint
-
-import (
-	"fmt"
-	"log"
-	"net"
-	"os"
-	"path/filepath"
-	"strconv"
-	"time"
-
-	crypticnet "cryptic-net"
-	"cryptic-net/garage"
-
-	"github.com/cryptic-io/pmux/pmuxlib"
-)
-
-func writeChildConf(
-	env *crypticnet.Env,
-	alloc crypticnet.DaemonYmlStorageAllocation,
-) (string, error) {
-
-	if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
-		return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
-	}
-
-	thisHost := env.Bootstrap.ThisHost()
-
-	peer := garage.Peer{
-		IP:        thisHost.Nebula.IP,
-		RPCPort:   alloc.RPCPort,
-		S3APIPort: alloc.S3APIPort,
-	}
-
-	pubKey, privKey := peer.RPCPeerKey()
-
-	nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
-	nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
-
-	if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
-		return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
-
-	} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
-		return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
-	}
-
-	garageTomlPath := filepath.Join(
-		env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
-	)
-
-	err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
-		MetaPath: alloc.MetaPath,
-		DataPath: alloc.DataPath,
-
-		RPCSecret: env.Bootstrap.GarageRPCSecret,
-
-		RPCAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
-		APIAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
-		WebAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.WebPort)),
-
-		BootstrapPeers: env.Bootstrap.GarageRPCPeerAddrs(),
-	})
-
-	if err != nil {
-		return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
-	}
-
-	return garageTomlPath, nil
-}
-
-func waitForArgs(env *crypticnet.Env, bin string, binArgs ...string) []string {
-
-	thisHost := env.Bootstrap.ThisHost()
-
-	var args []string
-
-	for _, alloc := range env.ThisDaemon().Storage.Allocations {
-		args = append(
-			args,
-			"wait-for",
-			net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
-			"--",
-		)
-	}
-
-	args = append(args, bin)
-	args = append(args, binArgs...)
-
-	return args
-}
-
-func Main() {
-
-	env, err := crypticnet.ReadEnv()
-
-	if err != nil {
-		log.Fatalf("reading envvars: %v", err)
-	}
-
-	var pmuxProcConfigs []pmuxlib.ProcessConfig
-
-	for _, alloc := range env.ThisDaemon().Storage.Allocations {
-
-		childConfPath, err := writeChildConf(env, alloc)
-
-		if err != nil {
-			log.Fatalf("writing child config file for alloc %+v: %v", alloc, err)
-		}
-
-		log.Printf("wrote config file %q", childConfPath)
-
-		pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
-			Name:        fmt.Sprintf("garage-%d", alloc.RPCPort),
-			Cmd:         "garage",
-			Args:        []string{"-c", childConfPath, "server"},
-			SigKillWait: 1 * time.Minute,
-		})
-
-	}
-
-	pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
-		Name:        "garage-apply-layout-diff",
-		Cmd:         "bash",
-		Args:        waitForArgs(env, "bash", "garage-apply-layout-diff"),
-		NoRestartOn: []int{0},
-	})
-
-	pmuxlib.Run(env.Context, pmuxlib.Config{Processes: pmuxProcConfigs})
-}

go-workspace/src/nebula/nebula.go

@@ -14,15 +14,6 @@ import (
 	"golang.org/x/crypto/curve25519"
 )
 
-// TODO this should one day not be hardcoded
-var ipCIDRMask = func() net.IPMask {
-	_, ipNet, err := net.ParseCIDR("10.10.0.0/16")
-	if err != nil {
-		panic(err)
-	}
-	return ipNet.Mask
-}()
-
 // HostCert contains the certificate and private key files which will need to
 // be present on a particular host. Each file is PEM encoded.
 type HostCert struct {
@@ -41,7 +32,7 @@ type CACert struct {
 // NewHostCert generates a new key/cert for a nebula host using the CA key
 // which will be found in the adminFS.
 func NewHostCert(
-	caCert CACert, hostName, hostIP string,
+	caCert CACert, hostName string, ip net.IP,
 ) (
 	HostCert, error,
 ) {
@@ -66,14 +57,9 @@ func NewHostCert(
 
 	expireAt := caCrt.Details.NotAfter.Add(-1 * time.Second)
 
-	ip := net.ParseIP(hostIP)
+	subnet := caCrt.Details.Subnets[0]
-	if ip == nil {
+	if !subnet.Contains(ip) {
-		return HostCert{}, fmt.Errorf("invalid host ip %q", hostIP)
+		return HostCert{}, fmt.Errorf("invalid ip %q, not contained by network subnet %q", ip, subnet)
-	}
-
-	ipNet := &net.IPNet{
-		IP:   ip,
-		Mask: ipCIDRMask,
 	}
 
 	var hostPub, hostKey []byte
@@ -89,7 +75,10 @@ func NewHostCert(
 	hostCrt := cert.NebulaCertificate{
 		Details: cert.NebulaCertificateDetails{
 			Name: hostName,
-			Ips:       []*net.IPNet{ipNet},
+			Ips: []*net.IPNet{{
+				IP:   ip,
+				Mask: subnet.Mask,
+			}},
 			NotBefore: time.Now(),
 			NotAfter:  expireAt,
 			PublicKey: hostPub,
@@ -122,7 +111,7 @@ func NewHostCert(
 
 // NewCACert generates a CACert. The domain should be the network's root domain,
 // and is included in the signing certificate's Name field.
-func NewCACert(domain string) (CACert, error) {
+func NewCACert(domain string, subnet *net.IPNet) (CACert, error) {
 
 	pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
 	if err != nil {
@@ -135,6 +124,7 @@ func NewCACert(domain string) (CACert, error) {
 	caCrt := cert.NebulaCertificate{
 		Details: cert.NebulaCertificateDetails{
 			Name:      fmt.Sprintf("%s cryptic-net root cert", domain),
+			Subnets:   []*net.IPNet{subnet},
 			NotBefore: now,
 			NotAfter:  expireAt,
 			PublicKey: pubKey,