Update daemon process tree diagram

Previously if the `daemon.yml` of a host was changed it would first
start up, load that new daemon.yml in, persist the new configuration for
the host to garage using `update-garage-host`, pull that config back
down and persist it to the bootstrap in `runDaemonPmuxOnce`, and restart
all child processes so they get the new config.

Now, once `daemon.yml` is loaded in we immediately merge it into and
persist this host's bootstrap file, prior to ever starting child
processes. This removes the necessity of restarting those process at
start.

This change also allows the bootstrap file to be the sole repository of
information required to pick a garage node to connect to, since it is
presumably always as up-to-date as it can possibly be. This allows for
removing some more logic from `Env`.

docs/dev/daemon-process-tree.plantuml

@@ -11,10 +11,12 @@ state AppDir {
         AppRun : * Set PATH to APPDIR/bin
     }
 
-    state "./bin/entrypoint" as entrypoint {
+    state "./bin/cryptic-net-main entrypoint daemon -c ./daemon.yml" as entrypoint {
-        entrypoint : * Merge given and default daemon.yml files
         entrypoint : * Create runtime dir at $_RUNTIME_DIR_PATH
         entrypoint : * Lock runtime dir
+        entrypoint : * Merge given and default daemon.yml files
+        entrypoint : * Copy bootstrap.tgz into $_DATA_DIR_PATH, if it's not there
+        entrypoint : * Merge daemon.yml config into bootstrap.tgz
         entrypoint : * Run child processes
     }
 
@@ -30,39 +32,36 @@ state AppDir {
     entrypoint --> dnsmasqEntrypoint : child
     dnsmasqEntrypoint --> dnsmasq : exec
 
-    state "./bin/nebula-entrypoint" as nebulaEntrypoint {
+    state "./bin/cryptic-net-main nebula-entrypoint" as nebulaEntrypoint {
         nebulaEntrypoint : * Create $_RUNTIME_DIR_PATH/nebula.yml
     }
 
     state "./bin/nebula -config $_RUNTIME_DIR_PATH/nebula.yml" as nebula
-    state "./bin/nebula-update-global-bucket" as nebulaUpdateGlobalBucket {
-        nebulaUpdateGlobalBucket : * Runs once then exits
-        nebulaUpdateGlobalBucket : * Updates network topo data in garage global bucket (used for bootstrapping)
-    }
 
     entrypoint --> nebulaEntrypoint : child
     nebulaEntrypoint --> nebula : exec
-    nebulaEntrypoint --> nebulaUpdateGlobalBucket : child
 
-    state "./bin/garage-entrypoint" as garageEntrypoint {
+    state "./bin/cryptic-net-main garage-entrypoint" as garageEntrypoint {
         garageEntrypoint : * Create $_RUNTIME_DIR_PATH/garage-N.toml\n  (one per storage allocation)
         garageEntrypoint : * Run child processes
     }
 
     state "./bin/garage -c $_RUNTIME_DIR_PATH/garage-N.toml server" as garage
-    state "./bin/garage-apply-layout-diff" as garageApplyLayoutDiff {
+    state "./bin/cryptic-net-main garage-apply-layout-diff" as garageApplyLayoutDiff {
         garageApplyLayoutDiff : * Runs once then exits
         garageApplyLayoutDiff : * Updates cluster topo
     }
-    state "./bin/garage-update-global-bucket" as garageUpdateGlobalBucket {
-        garageUpdateGlobalBucket : * Runs once then exits
-        garageUpdateGlobalBucket : * Updates cluster topo data in garage global bucket (used for bootstrapping)
-    }
 
     entrypoint --> garageEntrypoint : child (only if >1 storage allocation defined in daemon.yml)
     garageEntrypoint --> garage : child (one per storage allocation)
     garageEntrypoint --> garageApplyLayoutDiff : child
-    garageEntrypoint --> garageUpdateGlobalBucket : child
+
+    state "./bin/cryptic-net-main update-global-bucket" as updateGlobalBucket {
+        updateGlobalBucket : * Runs once then exits
+        updateGlobalBucket : * Updates the bootstrap data for the host in garage for other hosts to query
+    }
+
+    entrypoint --> updateGlobalBucket : child
 }
 
 @enduml

go-workspace/src/bootstrap/garage.go

@@ -2,6 +2,7 @@ package bootstrap
 
 import (
 	"cryptic-net/garage"
+	"fmt"
 )
 
 // Paths within the bootstrap FS related to garage.
@@ -45,3 +46,41 @@ func (b Bootstrap) GarageRPCPeerAddrs() []string {
 	}
 	return addrs
 }
+
+// ChooseGaragePeer returns a Peer for a garage instance from the network. It
+// will prefer a garage instance on this particular host, if there is one, but
+// will otherwise return a random endpoint.
+func (b Bootstrap) ChooseGaragePeer() garage.Peer {
+
+	thisHost := b.ThisHost()
+
+	if thisHost.Garage != nil && len(thisHost.Garage.Instances) > 0 {
+		inst := thisHost.Garage.Instances[0]
+		return garage.Peer{
+			IP:        thisHost.Nebula.IP,
+			RPCPort:   inst.RPCPort,
+			S3APIPort: inst.S3APIPort,
+		}
+	}
+
+	for _, peer := range b.GaragePeers() {
+		return peer
+	}
+
+	panic("no garage instances configured")
+}
+
+// GlobalBucketS3APIClient returns an S3 client pre-configured with access to
+// the global bucket.
+func (b Bootstrap) GlobalBucketS3APIClient() (garage.S3APIClient, error) {
+
+	addr := b.ChooseGaragePeer().S3APIAddr()
+	creds := b.GarageGlobalBucketS3APICredentials
+
+	client, err := garage.NewS3APIClient(addr, creds)
+	if err != nil {
+		return nil, fmt.Errorf("connecting to garage S3 API At %q: %w", addr, err)
+	}
+
+	return client, err
+}

go-workspace/src/bootstrap/hosts.go

@@ -1,7 +1,6 @@
 package bootstrap
 
 import (
-	"errors"
 	"fmt"
 	"io/fs"
 	"path/filepath"
@@ -42,62 +41,6 @@ type Host struct {
 	Garage *GarageHost `yaml:"garage,omitempty"`
 }
 
-func loadHostsLegacy(bootstrapFS fs.FS) (map[string]Host, error) {
-
-	hosts := map[string]Host{}
-
-	readAsYaml := func(into interface{}, path string) error {
-		b, err := fs.ReadFile(bootstrapFS, path)
-		if err != nil {
-			return fmt.Errorf("reading file from fs: %w", err)
-		}
-
-		return yaml.Unmarshal(b, into)
-	}
-
-	{
-		globPath := "nebula/hosts/*.yml"
-
-		nebulaHostFiles, err := fs.Glob(bootstrapFS, globPath)
-		if err != nil {
-			return nil, fmt.Errorf("listing nebula host files at %q in fs: %w", globPath, err)
-		}
-
-		for _, nebulaHostPath := range nebulaHostFiles {
-
-			hostName := filepath.Base(nebulaHostPath)
-			hostName = strings.TrimSuffix(hostName, filepath.Ext(hostName))
-
-			var nebulaHost NebulaHost
-			if err := readAsYaml(&nebulaHost, nebulaHostPath); err != nil {
-				return nil, fmt.Errorf("reading %q as yaml: %w", nebulaHostPath, err)
-			}
-
-			hosts[hostName] = Host{
-				Name:   hostName,
-				Nebula: nebulaHost,
-			}
-		}
-	}
-
-	for hostName, host := range hosts {
-
-		garageHostPath := filepath.Join("garage/hosts", hostName+".yml")
-
-		var garageHost GarageHost
-		if err := readAsYaml(&garageHost, garageHostPath); errors.Is(err, fs.ErrNotExist) {
-			continue
-		} else if err != nil {
-			return nil, fmt.Errorf("reading %q as yaml: %w", garageHostPath, err)
-		}
-
-		host.Garage = &garageHost
-		hosts[hostName] = host
-	}
-
-	return hosts, nil
-}
-
 func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
 
 	hosts := map[string]Host{}
@@ -131,18 +74,6 @@ func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
 		hosts[hostName] = host
 	}
 
-	if len(hosts) > 0 {
-		return hosts, nil
-	}
-
-	// We used to have the bootstrap file laid out differently. If no hosts were
-	// found then the bootstrap file is probably in that format.
-	hosts, err = loadHostsLegacy(bootstrapFS)
-
-	if err != nil {
-		return nil, fmt.Errorf("loading hosts in legacy layout from fs: %w", err)
-	}
-
 	if len(hosts) == 0 {
 		return nil, fmt.Errorf("failed to load any hosts from fs")
 	}

go-workspace/src/cmd/entrypoint/daemon.go

@@ -36,6 +36,9 @@ import (
 // * Merges the user-provided daemon.yml file with the default, and writes the
 // result to the runtime dir.
 //
+// * Merges daemon.yml configuration into the bootstrap configuration, and
+// rewrites the bootstrap file.
+//
 // * Sets up environment variables that all other sub-processes then use, based
 // on the runtime dir.
 //
@@ -79,7 +82,7 @@ func writeDaemonYml(userDaemonYmlPath, builtinDaemonYmlPath, runtimeDirPath stri
 	return nil
 }
 
-func writeBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
+func copyBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
 
 	path := env.DataDirBootstrapPath()
 	dirPath := filepath.Dir(path)
@@ -97,7 +100,7 @@ func writeBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
 	f.Close()
 
 	if err != nil {
-		return fmt.Errorf("writing new bootstrap file to %q: %w", path, err)
+		return fmt.Errorf("copying bootstrap file to %q: %w", path, err)
 	}
 
 	if err := env.LoadBootstrap(path); err != nil {
@@ -133,11 +136,11 @@ func reloadBootstrap(env *crypticnet.Env, s3Client garage.S3APIClient) (bool, er
 
 	buf := new(bytes.Buffer)
 	if err := env.Bootstrap.WithHosts(newHosts).WriteTo(buf); err != nil {
-		return false, fmt.Errorf("writing new bootstrap file: %w", err)
+		return false, fmt.Errorf("writing new bootstrap file to buffer: %w", err)
 	}
 
-	if err := writeBootstrapToDataDir(env, buf); err != nil {
+	if err := copyBootstrapToDataDir(env, buf); err != nil {
-		return false, fmt.Errorf("writing new bootstrap file: %w", err)
+		return false, fmt.Errorf("copying new bootstrap file to data dir: %w", err)
 	}
 
 	return true, nil
@@ -281,7 +284,7 @@ var subCmdDaemon = subCmd{
 
 		env := subCmdCtx.env
 
-		s3Client, err := env.GlobalBucketS3APIClient()
+		s3Client, err := env.Bootstrap.GlobalBucketS3APIClient()
 		if err != nil {
 			return fmt.Errorf("creating client for global bucket: %w", err)
 		}
@@ -349,7 +352,7 @@ var subCmdDaemon = subCmd{
 				return fmt.Errorf("opening file %q: %w", env.BootstrapPath, err)
 			}
 
-			err = writeBootstrapToDataDir(env, f)
+			err = copyBootstrapToDataDir(env, f)
 			f.Close()
 
 			if err != nil {
@@ -361,6 +364,46 @@ var subCmdDaemon = subCmd{
 			return fmt.Errorf("generating daemon.yml file: %w", err)
 		}
 
+		{
+			// we update this Host's data using whatever configuration has been
+			// provided by daemon.yml. This way the daemon has the most
+			// up-to-date possible bootstrap. This updated bootstrap will later
+			// get updated in garage using update-global-bucket, so other hosts
+			// will see it as well.
+
+			// ThisDaemon can only be called after writeDaemonYml.
+			daemon := env.ThisDaemon()
+			host := env.Bootstrap.ThisHost()
+
+			host.Nebula.PublicAddr = daemon.VPN.PublicAddr
+
+			host.Garage = nil
+
+			if allocs := daemon.Storage.Allocations; len(allocs) > 0 {
+
+				host.Garage = new(bootstrap.GarageHost)
+
+				for _, alloc := range allocs {
+					host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
+						RPCPort:   alloc.RPCPort,
+						S3APIPort: alloc.S3APIPort,
+						WebPort:   alloc.WebPort,
+					})
+				}
+			}
+
+			env.Bootstrap.Hosts[host.Name] = host
+
+			buf := new(bytes.Buffer)
+			if err := env.Bootstrap.WithHosts(env.Bootstrap.Hosts).WriteTo(buf); err != nil {
+				return fmt.Errorf("writing new bootstrap file to buffer: %w", err)
+			}
+
+			if err := copyBootstrapToDataDir(env, buf); err != nil {
+				return fmt.Errorf("copying new bootstrap file to data dir: %w", err)
+			}
+		}
+
 		for key, val := range env.ToMap() {
 			if err := os.Setenv(key, val); err != nil {
 				return fmt.Errorf("failed to set %q to %q: %w", key, val, err)

go-workspace/src/cmd/entrypoint/garage.go

@@ -30,7 +30,7 @@ var subCmdGarageMC = subCmd{
 
 		env := subCmdCtx.env
 
-		s3APIAddr := env.ChooseGaragePeer().S3APIAddr()
+		s3APIAddr := env.Bootstrap.ChooseGaragePeer().S3APIAddr()
 
 		if *keyID == "" || *keySecret == "" {
 
@@ -90,7 +90,7 @@ var subCmdGarageCLI = subCmd{
 			args    = append([]string{"garage"}, subCmdCtx.args...)
 			cliEnv  = append(
 				os.Environ(),
-				"GARAGE_RPC_HOST="+env.ChooseGaragePeer().RPCAddr(),
+				"GARAGE_RPC_HOST="+env.Bootstrap.ChooseGaragePeer().RPCAddr(),
 				"GARAGE_RPC_SECRET="+env.Bootstrap.GarageRPCSecret,
 			)
 		)

go-workspace/src/cmd/entrypoint/hosts.go

@@ -64,7 +64,7 @@ var subCmdHostsAdd = subCmd{
 
 		env := subCmdCtx.env
 
-		client, err := env.GlobalBucketS3APIClient()
+		client, err := env.Bootstrap.GlobalBucketS3APIClient()
 		if err != nil {
 			return fmt.Errorf("creating client for global bucket: %w", err)
 		}
@@ -88,7 +88,7 @@ var subCmdHostsList = subCmd{
 
 		env := subCmdCtx.env
 
-		client, err := env.GlobalBucketS3APIClient()
+		client, err := env.Bootstrap.GlobalBucketS3APIClient()
 		if err != nil {
 			return fmt.Errorf("creating client for global bucket: %w", err)
 		}
@@ -132,7 +132,7 @@ var subCmdHostsDelete = subCmd{
 
 		env := subCmdCtx.env
 
-		client, err := env.GlobalBucketS3APIClient()
+		client, err := env.Bootstrap.GlobalBucketS3APIClient()
 		if err != nil {
 			return fmt.Errorf("creating client for global bucket: %w", err)
 		}
@@ -195,7 +195,7 @@ var subCmdHostsMakeBootstrap = subCmd{
 			return fmt.Errorf("reading admin.tgz with --admin-path of %q: %w", *adminPath, err)
 		}
 
-		client, err := env.GlobalBucketS3APIClient()
+		client, err := env.Bootstrap.GlobalBucketS3APIClient()
 		if err != nil {
 			return fmt.Errorf("creating client for global bucket: %w", err)
 		}

go-workspace/src/cmd/update-global-bucket/main.go

@@ -13,42 +13,18 @@ func Main() {
 		log.Fatalf("reading envvars: %v", err)
 	}
 
-	client, err := env.GlobalBucketS3APIClient()
+	client, err := env.Bootstrap.GlobalBucketS3APIClient()
 	if err != nil {
 		log.Fatalf("creating client for global bucket: %v", err)
 	}
 
-	host := env.Bootstrap.ThisHost()
+	err = bootstrap.PutGarageBoostrapHost(
+		env.Context,
+		client,
+		env.Bootstrap.ThisHost(),
+	)
 
-	// We update the Host for this host in place, prior to writing it via the
+	if err != nil {
-	// bootstrap method. We want to ensure that any changes made via daemon are
-	// reflected into the bootstrap data which is pushed up.
-	//
-	// TODO it'd be better if this was done within the daemon command itself,
-	// prior to any sub-processes being started. This would help us avoid this
-	// weird logic here, and would prevent all sub-processes from needing to be
-	// restarted the first time the daemon is started after daemon.yml is
-	// modified.
-	daemon := env.ThisDaemon()
-
-	host.Nebula.PublicAddr = daemon.VPN.PublicAddr
-
-	host.Garage = nil
-
-	if allocs := daemon.Storage.Allocations; len(allocs) > 0 {
-
-		host.Garage = new(bootstrap.GarageHost)
-
-		for _, alloc := range allocs {
-			host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
-				RPCPort:   alloc.RPCPort,
-				S3APIPort: alloc.S3APIPort,
-				WebPort:   alloc.WebPort,
-			})
-		}
-	}
-
-	if err := bootstrap.PutGarageBoostrapHost(env.Context, client, host); err != nil {
 		log.Fatal(err)
 	}
 }

go-workspace/src/garage.go

@@ -1,44 +0,0 @@
-package crypticnet
-
-import (
-	"cryptic-net/garage"
-	"fmt"
-)
-
-// ChooseGaragePeer returns a Peer for a garage instance from the network. It
-// will prefer a garage instance on this particular host, if there is one, but
-// will otherwise return a random endpoint.
-func (env *Env) ChooseGaragePeer() garage.Peer {
-
-	// TODO this only works well within the daemon process, otherwise daemon.yml
-	// isn't available.
-	if allocs := env.ThisDaemon().Storage.Allocations; len(allocs) > 0 {
-
-		return garage.Peer{
-			IP:        env.Bootstrap.ThisHost().Nebula.IP,
-			RPCPort:   allocs[0].RPCPort,
-			S3APIPort: allocs[0].S3APIPort,
-		}
-	}
-
-	for _, peer := range env.Bootstrap.GaragePeers() {
-		return peer
-	}
-
-	panic("no garage instances configured")
-}
-
-// GlobalBucketS3APIClient returns an S3 client pre-configured with access to
-// the global bucket.
-func (env *Env) GlobalBucketS3APIClient() (garage.S3APIClient, error) {
-
-	addr := env.ChooseGaragePeer().S3APIAddr()
-	creds := env.Bootstrap.GarageGlobalBucketS3APICredentials
-
-	client, err := garage.NewS3APIClient(addr, creds)
-	if err != nil {
-		return nil, fmt.Errorf("connecting to garage S3 API At %q: %w", addr, err)
-	}
-
-	return client, err
-}