Update host's config in bootstrap using daemon.yml prior to starting pmux

Previously if the `daemon.yml` of a host was changed it would first
start up, load that new daemon.yml in, persist the new configuration for
the host to garage using `update-garage-host`, pull that config back
down and persist it to the bootstrap in `runDaemonPmuxOnce`, and restart
all child processes so they get the new config.

Now, once `daemon.yml` is loaded in we immediately merge it into and
persist this host's bootstrap file, prior to ever starting child
processes. This removes the necessity of restarting those process at
start.

This change also allows the bootstrap file to be the sole repository of
information required to pick a garage node to connect to, since it is
presumably always as up-to-date as it can possibly be. This allows for
removing some more logic from `Env`.

go-workspace/src/bootstrap/garage.go

@@ -2,6 +2,7 @@ package bootstrap
 
 import (
 	"cryptic-net/garage"
+	"fmt"
 )
 
 // Paths within the bootstrap FS related to garage.
@@ -45,3 +46,41 @@ func (b Bootstrap) GarageRPCPeerAddrs() []string {
 	}
 	return addrs
 }
+
+// ChooseGaragePeer returns a Peer for a garage instance from the network. It
+// will prefer a garage instance on this particular host, if there is one, but
+// will otherwise return a random endpoint.
+func (b Bootstrap) ChooseGaragePeer() garage.Peer {
+
+	thisHost := b.ThisHost()
+
+	if thisHost.Garage != nil && len(thisHost.Garage.Instances) > 0 {
+		inst := thisHost.Garage.Instances[0]
+		return garage.Peer{
+			IP:        thisHost.Nebula.IP,
+			RPCPort:   inst.RPCPort,
+			S3APIPort: inst.S3APIPort,
+		}
+	}
+
+	for _, peer := range b.GaragePeers() {
+		return peer
+	}
+
+	panic("no garage instances configured")
+}
+
+// GlobalBucketS3APIClient returns an S3 client pre-configured with access to
+// the global bucket.
+func (b Bootstrap) GlobalBucketS3APIClient() (garage.S3APIClient, error) {
+
+	addr := b.ChooseGaragePeer().S3APIAddr()
+	creds := b.GarageGlobalBucketS3APICredentials
+
+	client, err := garage.NewS3APIClient(addr, creds)
+	if err != nil {
+		return nil, fmt.Errorf("connecting to garage S3 API At %q: %w", addr, err)
+	}
+
+	return client, err
+}

go-workspace/src/cmd/entrypoint/daemon.go

@@ -36,6 +36,9 @@ import (
 // * Merges the user-provided daemon.yml file with the default, and writes the
 // result to the runtime dir.
 //
+// * Merges daemon.yml configuration into the bootstrap configuration, and
+// rewrites the bootstrap file.
+//
 // * Sets up environment variables that all other sub-processes then use, based
 // on the runtime dir.
 //
@@ -79,7 +82,7 @@ func writeDaemonYml(userDaemonYmlPath, builtinDaemonYmlPath, runtimeDirPath stri
 	return nil
 }
 
-func writeBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
+func copyBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
 
 	path := env.DataDirBootstrapPath()
 	dirPath := filepath.Dir(path)
@@ -97,7 +100,7 @@ func writeBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
 	f.Close()
 
 	if err != nil {
-		return fmt.Errorf("writing new bootstrap file to %q: %w", path, err)
+		return fmt.Errorf("copying bootstrap file to %q: %w", path, err)
 	}
 
 	if err := env.LoadBootstrap(path); err != nil {
@@ -133,11 +136,11 @@ func reloadBootstrap(env *crypticnet.Env, s3Client garage.S3APIClient) (bool, er
 
 	buf := new(bytes.Buffer)
 	if err := env.Bootstrap.WithHosts(newHosts).WriteTo(buf); err != nil {
-		return false, fmt.Errorf("writing new bootstrap file: %w", err)
+		return false, fmt.Errorf("writing new bootstrap file to buffer: %w", err)
 	}
 
-	if err := writeBootstrapToDataDir(env, buf); err != nil {
+	if err := copyBootstrapToDataDir(env, buf); err != nil {
-		return false, fmt.Errorf("writing new bootstrap file: %w", err)
+		return false, fmt.Errorf("copying new bootstrap file to data dir: %w", err)
 	}
 
 	return true, nil
@@ -281,7 +284,7 @@ var subCmdDaemon = subCmd{
 
 		env := subCmdCtx.env
 
-		s3Client, err := env.GlobalBucketS3APIClient()
+		s3Client, err := env.Bootstrap.GlobalBucketS3APIClient()
 		if err != nil {
 			return fmt.Errorf("creating client for global bucket: %w", err)
 		}
@@ -349,7 +352,7 @@ var subCmdDaemon = subCmd{
 				return fmt.Errorf("opening file %q: %w", env.BootstrapPath, err)
 			}
 
-			err = writeBootstrapToDataDir(env, f)
+			err = copyBootstrapToDataDir(env, f)
 			f.Close()
 
 			if err != nil {
@@ -361,6 +364,46 @@ var subCmdDaemon = subCmd{
 			return fmt.Errorf("generating daemon.yml file: %w", err)
 		}
 
+		{
+			// we update this Host's data using whatever configuration has been
+			// provided by daemon.yml. This way the daemon has the most
+			// up-to-date possible bootstrap. This updated bootstrap will later
+			// get updated in garage using update-global-bucket, so other hosts
+			// will see it as well.
+
+			// ThisDaemon can only be called after writeDaemonYml.
+			daemon := env.ThisDaemon()
+			host := env.Bootstrap.ThisHost()
+
+			host.Nebula.PublicAddr = daemon.VPN.PublicAddr
+
+			host.Garage = nil
+
+			if allocs := daemon.Storage.Allocations; len(allocs) > 0 {
+
+				host.Garage = new(bootstrap.GarageHost)
+
+				for _, alloc := range allocs {
+					host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
+						RPCPort:   alloc.RPCPort,
+						S3APIPort: alloc.S3APIPort,
+						WebPort:   alloc.WebPort,
+					})
+				}
+			}
+
+			env.Bootstrap.Hosts[host.Name] = host
+
+			buf := new(bytes.Buffer)
+			if err := env.Bootstrap.WithHosts(env.Bootstrap.Hosts).WriteTo(buf); err != nil {
+				return fmt.Errorf("writing new bootstrap file to buffer: %w", err)
+			}
+
+			if err := copyBootstrapToDataDir(env, buf); err != nil {
+				return fmt.Errorf("copying new bootstrap file to data dir: %w", err)
+			}
+		}
+
 		for key, val := range env.ToMap() {
 			if err := os.Setenv(key, val); err != nil {
 				return fmt.Errorf("failed to set %q to %q: %w", key, val, err)

go-workspace/src/cmd/entrypoint/garage.go

@@ -30,7 +30,7 @@ var subCmdGarageMC = subCmd{
 
 		env := subCmdCtx.env
 
-		s3APIAddr := env.ChooseGaragePeer().S3APIAddr()
+		s3APIAddr := env.Bootstrap.ChooseGaragePeer().S3APIAddr()
 
 		if *keyID == "" || *keySecret == "" {
 
@@ -90,7 +90,7 @@ var subCmdGarageCLI = subCmd{
 			args    = append([]string{"garage"}, subCmdCtx.args...)
 			cliEnv  = append(
 				os.Environ(),
-				"GARAGE_RPC_HOST="+env.ChooseGaragePeer().RPCAddr(),
+				"GARAGE_RPC_HOST="+env.Bootstrap.ChooseGaragePeer().RPCAddr(),
 				"GARAGE_RPC_SECRET="+env.Bootstrap.GarageRPCSecret,
 			)
 		)

go-workspace/src/cmd/entrypoint/hosts.go

@@ -64,7 +64,7 @@ var subCmdHostsAdd = subCmd{
 
 		env := subCmdCtx.env
 
-		client, err := env.GlobalBucketS3APIClient()
+		client, err := env.Bootstrap.GlobalBucketS3APIClient()
 		if err != nil {
 			return fmt.Errorf("creating client for global bucket: %w", err)
 		}
@@ -88,7 +88,7 @@ var subCmdHostsList = subCmd{
 
 		env := subCmdCtx.env
 
-		client, err := env.GlobalBucketS3APIClient()
+		client, err := env.Bootstrap.GlobalBucketS3APIClient()
 		if err != nil {
 			return fmt.Errorf("creating client for global bucket: %w", err)
 		}
@@ -132,7 +132,7 @@ var subCmdHostsDelete = subCmd{
 
 		env := subCmdCtx.env
 
-		client, err := env.GlobalBucketS3APIClient()
+		client, err := env.Bootstrap.GlobalBucketS3APIClient()
 		if err != nil {
 			return fmt.Errorf("creating client for global bucket: %w", err)
 		}
@@ -195,7 +195,7 @@ var subCmdHostsMakeBootstrap = subCmd{
 			return fmt.Errorf("reading admin.tgz with --admin-path of %q: %w", *adminPath, err)
 		}
 
-		client, err := env.GlobalBucketS3APIClient()
+		client, err := env.Bootstrap.GlobalBucketS3APIClient()
 		if err != nil {
 			return fmt.Errorf("creating client for global bucket: %w", err)
 		}

go-workspace/src/cmd/update-global-bucket/main.go

@@ -13,42 +13,18 @@ func Main() {
 		log.Fatalf("reading envvars: %v", err)
 	}
 
-	client, err := env.GlobalBucketS3APIClient()
+	client, err := env.Bootstrap.GlobalBucketS3APIClient()
 	if err != nil {
 		log.Fatalf("creating client for global bucket: %v", err)
 	}
 
-	host := env.Bootstrap.ThisHost()
+	err = bootstrap.PutGarageBoostrapHost(
+		env.Context,
+		client,
+		env.Bootstrap.ThisHost(),
+	)
 
-	// We update the Host for this host in place, prior to writing it via the
+	if err != nil {
-	// bootstrap method. We want to ensure that any changes made via daemon are
-	// reflected into the bootstrap data which is pushed up.
-	//
-	// TODO it'd be better if this was done within the daemon command itself,
-	// prior to any sub-processes being started. This would help us avoid this
-	// weird logic here, and would prevent all sub-processes from needing to be
-	// restarted the first time the daemon is started after daemon.yml is
-	// modified.
-	daemon := env.ThisDaemon()
-
-	host.Nebula.PublicAddr = daemon.VPN.PublicAddr
-
-	host.Garage = nil
-
-	if allocs := daemon.Storage.Allocations; len(allocs) > 0 {
-
-		host.Garage = new(bootstrap.GarageHost)
-
-		for _, alloc := range allocs {
-			host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
-				RPCPort:   alloc.RPCPort,
-				S3APIPort: alloc.S3APIPort,
-				WebPort:   alloc.WebPort,
-			})
-		}
-	}
-
-	if err := bootstrap.PutGarageBoostrapHost(env.Context, client, host); err != nil {
 		log.Fatal(err)
 	}
 }

go-workspace/src/garage.go

@@ -1,44 +0,0 @@
-package crypticnet
-
-import (
-	"cryptic-net/garage"
-	"fmt"
-)
-
-// ChooseGaragePeer returns a Peer for a garage instance from the network. It
-// will prefer a garage instance on this particular host, if there is one, but
-// will otherwise return a random endpoint.
-func (env *Env) ChooseGaragePeer() garage.Peer {
-
-	// TODO this only works well within the daemon process, otherwise daemon.yml
-	// isn't available.
-	if allocs := env.ThisDaemon().Storage.Allocations; len(allocs) > 0 {
-
-		return garage.Peer{
-			IP:        env.Bootstrap.ThisHost().Nebula.IP,
-			RPCPort:   allocs[0].RPCPort,
-			S3APIPort: allocs[0].S3APIPort,
-		}
-	}
-
-	for _, peer := range env.Bootstrap.GaragePeers() {
-		return peer
-	}
-
-	panic("no garage instances configured")
-}
-
-// GlobalBucketS3APIClient returns an S3 client pre-configured with access to
-// the global bucket.
-func (env *Env) GlobalBucketS3APIClient() (garage.S3APIClient, error) {
-
-	addr := env.ChooseGaragePeer().S3APIAddr()
-	creds := env.Bootstrap.GarageGlobalBucketS3APICredentials
-
-	client, err := garage.NewS3APIClient(addr, creds)
-	if err != nil {
-		return nil, fmt.Errorf("connecting to garage S3 API At %q: %w", addr, err)
-	}
-
-	return client, err
-}