Refactor children process reloading, add garage reloading

default.nix

@@ -1,14 +1,10 @@
 {
-
   buildSystem ? builtins.currentSystem,
   hostSystem ? buildSystem,
   pkgsNix ? (import ./nix/pkgs.nix),
 
   revision ? "dev",
   releaseName ? "dev",
-
-  bootstrap ? null, # TODO remove this
-
 }: let
 
   pkgs = pkgsNix.default {
@@ -94,43 +90,31 @@ in rec {
 
   };
 
-  rootedBootstrap = pkgs.stdenv.mkDerivation {
-    name = "isle-rooted-bootstrap";
-
-    src = bootstrap;
-
-    builder = builtins.toFile "builder.sh" ''
-      source $stdenv/setup
-      mkdir -p "$out"/share
-      cp "$src" "$out"/share/bootstrap.json
-    '';
-  };
-
-  appDir = pkgs.stdenv.mkDerivation {
-    name = "isle-AppDir";
-
-    src = pkgs.buildEnv {
+  appDirBase = pkgs.buildEnv {
     name = "isle-AppDir-base";
     paths = [
-
       ./AppDir
       version
       dnsmasq
       nebula
       garage
       pkgs.minio-client
-        goBinaries
-
-      ] ++ (if bootstrap != null then [ rootedBootstrap ] else []);
+    ];
   };
 
+  appDir = pkgs.stdenv.mkDerivation {
+    name = "isle-AppDir";
+
+    src = appDirBase;
+    inherit goBinaries;
+
     builder = builtins.toFile "build.sh" ''
       source $stdenv/setup
       cp -rL "$src" "$out"
       chmod +w "$out" -R
 
       cd "$out"
-      cp ./bin/entrypoint ./AppRun
+      cp $goBinaries/bin/entrypoint ./AppRun
     '';
   };
 
@@ -178,4 +162,23 @@ in rec {
     export SHELL=${pkgs.bash}/bin/bash
     exec ${pkgs.bash}/bin/bash ${./tests}/entrypoint.sh "$@"
   '';
+
+  devShell = pkgs.mkShell {
+    buildInputs = [
+      pkgs.go
+      pkgs.golangci-lint
+      pkgs.gopls
+      (pkgs.callPackage ./nix/gowrap.nix {})
+    ];
+    shellHook = ''
+      true # placeholder
+    '';
+  };
+
+  testShell = pkgs.mkShell {
+    APPDIR = appDirBase;
+    buildInputs = [
+      pkgs.go
+    ];
+  };
 }

go/daemon/children/children.go

@@ -10,6 +10,7 @@ import (
 	"os"
 
 	"code.betamike.com/micropelago/pmux/pmuxlib"
+	"dev.mediocregopher.com/mediocre-go-lib.git/mctx"
 	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
 
 	"isle/bootstrap"
@@ -49,10 +50,12 @@ func (o *Opts) withDefaults() *Opts {
 // - garage (0 or more, depending on configured storage allocations)
 type Children struct {
 	logger           *mlog.Logger
-	networkConfig daecommon.NetworkConfig
 	runtimeDir       toolkit.Dir
+	garageAdminToken string
 	opts             Opts
 
+	garageRPCSecret string
+
 	pmux *pmuxlib.Pmux
 }
 
@@ -81,9 +84,10 @@ func New(
 
 	c := &Children{
 		logger:           logger,
-		networkConfig: networkConfig,
 		runtimeDir:       runtimeDir,
+		garageAdminToken: garageAdminToken,
 		opts:             *opts,
+		garageRPCSecret:  garageRPCSecret,
 	}
 
 	pmuxConfig, err := c.newPmuxConfig(
@@ -112,57 +116,126 @@ func New(
 	return c, nil
 }
 
-// RestartDNSMasq rewrites the dnsmasq config and restarts the process.
-//
 // TODO block until process has been confirmed to have come back up
 // successfully.
-func (c *Children) RestartDNSMasq(hostBootstrap bootstrap.Bootstrap) error {
-	_, err := dnsmasqWriteConfig(
-		c.runtimeDir.Path, c.networkConfig, hostBootstrap,
-	)
-	if err != nil {
+func (c *Children) reloadDNSMasq(
+	ctx context.Context,
+	networkConfig daecommon.NetworkConfig,
+	hostBootstrap bootstrap.Bootstrap,
+) error {
+	if _, changed, err := dnsmasqWriteConfig(
+		ctx, c.logger, c.runtimeDir.Path, networkConfig, hostBootstrap,
+	); err != nil {
 		return fmt.Errorf("writing new dnsmasq config: %w", err)
-	}
-
-	c.pmux.Restart("dnsmasq")
+	} else if !changed {
+		c.logger.Info(ctx, "No changes to dnsmasq config file")
 		return nil
 	}
 
-// RestartNebula rewrites the nebula config and restarts the process.
-//
-// TODO block until process has been confirmed to have come back up
-// successfully.
-func (c *Children) RestartNebula(hostBootstrap bootstrap.Bootstrap) error {
-	_, err := nebulaWriteConfig(
-		c.runtimeDir.Path, c.networkConfig, hostBootstrap,
-	)
-	if err != nil {
-		return fmt.Errorf("writing a new nebula config: %w", err)
+	c.logger.Info(ctx, "dnsmasq config file has changed, restarting process")
+	c.pmux.Restart("dnsmasq")
+
+	return nil
 }
 
+func (c *Children) reloadNebula(
+	ctx context.Context,
+	networkConfig daecommon.NetworkConfig,
+	hostBootstrap bootstrap.Bootstrap,
+) error {
+	if _, changed, err := nebulaWriteConfig(
+		ctx, c.logger, c.runtimeDir.Path, networkConfig, hostBootstrap,
+	); err != nil {
+		return fmt.Errorf("writing a new nebula config: %w", err)
+	} else if !changed {
+		c.logger.Info(ctx, "No changes to nebula config file")
+		return nil
+	}
+
+	c.logger.Info(ctx, "nebula config file has changed, restarting process")
 	c.pmux.Restart("nebula")
+
+	if err := waitForNebula(ctx, c.logger, hostBootstrap); err != nil {
+		return fmt.Errorf("waiting for nebula to start: %w", err)
+	}
+
+	return nil
+}
+
+// TODO this doesn't handle removing garage nodes
+func (c *Children) reloadGarage(
+	ctx context.Context,
+	networkConfig daecommon.NetworkConfig,
+	hostBootstrap bootstrap.Bootstrap,
+) error {
+	allocs := networkConfig.Storage.Allocations
+	if len(allocs) == 0 {
+		return nil
+	}
+
+	var anyChanged bool
+	for _, alloc := range allocs {
+		var (
+			procName = garagePmuxProcName(alloc)
+			ctx      = mctx.Annotate(
+				ctx,
+				"garageProcName", procName,
+				"garageDataPath", alloc.DataPath,
+			)
+		)
+
+		_, changed, err := garageWriteChildConfig(
+			ctx,
+			c.logger,
+			c.garageRPCSecret,
+			c.runtimeDir.Path,
+			c.garageAdminToken,
+			hostBootstrap,
+			alloc,
+		)
+		if err != nil {
+			return fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
+		} else if !changed {
+			c.logger.Info(ctx, "No changes to garage config file")
+			continue
+		}
+
+		anyChanged = true
+
+		c.logger.Info(ctx, "garage config has changed, restarting process")
+		c.pmux.Restart(procName)
+	}
+
+	if anyChanged {
+		if err := waitForGarage(
+			ctx, c.logger, networkConfig, c.garageAdminToken, hostBootstrap,
+		); err != nil {
+			return fmt.Errorf("waiting for garage to start: %w", err)
+		}
+	}
+
 	return nil
 }
 
 // Reload applies a ReloadDiff to the Children, using the given bootstrap which
 // must be the same one which was passed to CalculateReloadDiff.
 func (c *Children) Reload(
-	ctx context.Context, newBootstrap bootstrap.Bootstrap, diff ReloadDiff,
+	ctx context.Context,
+	newNetworkConfig daecommon.NetworkConfig,
+	newBootstrap bootstrap.Bootstrap,
 ) error {
+	if err := c.reloadNebula(ctx, newNetworkConfig, newBootstrap); err != nil {
+		return fmt.Errorf("reloading nebula: %w", err)
+	}
+
 	var errs []error
 
-	if diff.DNSChanged {
-		c.logger.Info(ctx, "Restarting dnsmasq to account for bootstrap changes")
-		if err := c.RestartDNSMasq(newBootstrap); err != nil {
-			errs = append(errs, fmt.Errorf("restarting dnsmasq: %w", err))
-		}
+	if err := c.reloadDNSMasq(ctx, newNetworkConfig, newBootstrap); err != nil {
+		errs = append(errs, fmt.Errorf("reloading dnsmasq: %w", err))
 	}
 
-	if diff.NebulaChanged {
-		c.logger.Info(ctx, "Restarting nebula to account for bootstrap changes")
-		if err := c.RestartNebula(newBootstrap); err != nil {
-			errs = append(errs, fmt.Errorf("restarting nebula: %w", err))
-		}
+	if err := c.reloadGarage(ctx, newNetworkConfig, newBootstrap); err != nil {
+		errs = append(errs, fmt.Errorf("reloading garage: %w", err))
 	}
 
 	return errors.Join(errs...)

go/daemon/children/diff.go

@@ -1,47 +0,0 @@
-package children
-
-import (
-	"errors"
-	"fmt"
-	"isle/bootstrap"
-	"isle/daemon/daecommon"
-	"reflect"
-)
-
-// ReloadDiff describes which children had their configurations changed as part
-// of a change in the bootstrap.
-type ReloadDiff struct {
-	NebulaChanged bool
-	DNSChanged    bool
-}
-
-// CalculateReloadDiff calculates a ReloadDiff based on an old and new
-// bootstrap.
-func CalculateReloadDiff(
-	networkConfig daecommon.NetworkConfig,
-	prevBootstrap, nextBootstrap bootstrap.Bootstrap,
-) (
-	diff ReloadDiff, err error,
-) {
-	{
-		prevNebulaConfig, prevErr := nebulaConfig(networkConfig, prevBootstrap)
-		nextNebulaConfig, nextErr := nebulaConfig(networkConfig, nextBootstrap)
-		if err = errors.Join(prevErr, nextErr); err != nil {
-			err = fmt.Errorf("calculating nebula config: %w", err)
-			return
-		}
-
-		diff.NebulaChanged = !reflect.DeepEqual(
-			prevNebulaConfig, nextNebulaConfig,
-		)
-	}
-
-	{
-		diff.DNSChanged = !reflect.DeepEqual(
-			dnsmasqConfig(networkConfig, prevBootstrap),
-			dnsmasqConfig(networkConfig, nextBootstrap),
-		)
-	}
-
-	return
-}

go/daemon/children/dnsmasq.go

@@ -7,55 +7,50 @@ import (
 	"isle/daemon/daecommon"
 	"isle/dnsmasq"
 	"path/filepath"
-	"sort"
 
 	"code.betamike.com/micropelago/pmux/pmuxlib"
 	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
 )
 
-func dnsmasqConfig(
-	networkConfig daecommon.NetworkConfig, hostBootstrap bootstrap.Bootstrap,
-) dnsmasq.ConfData {
-	hostsSlice := make([]dnsmasq.ConfDataHost, 0, len(hostBootstrap.Hosts))
+func dnsmasqWriteConfig(
+	ctx context.Context,
+	logger *mlog.Logger,
+	runtimeDirPath string,
+	networkConfig daecommon.NetworkConfig,
+	hostBootstrap bootstrap.Bootstrap,
+) (
+	string, bool, error,
+) {
+	hosts := make([]dnsmasq.ConfDataHost, 0, len(hostBootstrap.Hosts))
 	for _, host := range hostBootstrap.Hosts {
-		hostsSlice = append(hostsSlice, dnsmasq.ConfDataHost{
+		hosts = append(hosts, dnsmasq.ConfDataHost{
 			Name: string(host.Name),
 			IP:   host.IP().String(),
 		})
 	}
 
-	sort.Slice(hostsSlice, func(i, j int) bool {
-		return hostsSlice[i].IP < hostsSlice[j].IP
-	})
-
-	return dnsmasq.ConfData{
+	var (
+		confPath = filepath.Join(runtimeDirPath, "dnsmasq.conf")
+		confData = dnsmasq.ConfData{
 			Resolvers: networkConfig.DNS.Resolvers,
 			Domain:    hostBootstrap.NetworkCreationParams.Domain,
 			IP:        hostBootstrap.ThisHost().IP().String(),
-		Hosts:     hostsSlice,
+			Hosts:     hosts,
 		}
-}
-
-func dnsmasqWriteConfig(
-	runtimeDirPath string,
-	networkConfig daecommon.NetworkConfig,
-	hostBootstrap bootstrap.Bootstrap,
-) (
-	string, error,
-) {
-	var (
-		confPath = filepath.Join(runtimeDirPath, "dnsmasq.conf")
-		confData = dnsmasqConfig(networkConfig, hostBootstrap)
 	)
 
-	if err := dnsmasq.WriteConfFile(confPath, confData); err != nil {
-		return "", fmt.Errorf("writing dnsmasq.conf to %q: %w", confPath, err)
+	changed, err := dnsmasq.WriteConfFile(ctx, logger, confPath, confData)
+	if err != nil {
+		return "", false, fmt.Errorf(
+			"writing dnsmasq.conf to %q: %w", confPath, err,
+		)
 	}
 
-	return confPath, nil
+	return confPath, changed, nil
 }
 
 func dnsmasqPmuxProcConfig(
+	ctx context.Context,
 	logger *mlog.Logger,
 	runtimeDirPath, binDirPath string,
 	networkConfig daecommon.NetworkConfig,
@@ -63,8 +58,8 @@ func dnsmasqPmuxProcConfig(
 ) (
 	pmuxlib.ProcessConfig, error,
 ) {
-	confPath, err := dnsmasqWriteConfig(
-		runtimeDirPath, networkConfig, hostBootstrap,
+	confPath, _, err := dnsmasqWriteConfig(
+		ctx, logger, runtimeDirPath, networkConfig, hostBootstrap,
 	)
 	if err != nil {
 		return pmuxlib.ProcessConfig{}, fmt.Errorf(

go/daemon/children/garage.go

@@ -2,6 +2,7 @@ package children
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"isle/bootstrap"
 	"isle/daemon/daecommon"
@@ -29,9 +30,6 @@ func waitForGarage(
 ) error {
 
 	allocs := networkConfig.Storage.Allocations
-
-	// if this host doesn't have any allocations specified then fall back to
-	// waiting for nebula
 	if len(allocs) == 0 {
 		return nil
 	}
@@ -49,12 +47,16 @@ func waitForGarage(
 			adminClientLogger, adminAddr, adminToken,
 		)
 
-		ctx := mctx.Annotate(ctx, "garageAdminAddr", adminAddr)
-		logger.Debug(ctx, "waiting for garage instance to start")
+		ctx := mctx.Annotate(
+			ctx, "garageAdminAddr", adminAddr, "garageDataPath", alloc.DataPath,
+		)
+		logger.Info(ctx, "Waiting for garage instance to be healthy")
 
 		if err := adminClient.Wait(ctx); err != nil {
 			return fmt.Errorf("waiting for garage instance %q to start up: %w", adminAddr, err)
 		}
+
+		adminClient.Close()
 	}
 
 	return nil
@@ -62,17 +64,19 @@ func waitForGarage(
 }
 
 func garageWriteChildConfig(
+	ctx context.Context,
+	logger *mlog.Logger,
 	rpcSecret, runtimeDirPath, adminToken string,
 	hostBootstrap bootstrap.Bootstrap,
 	alloc daecommon.ConfigStorageAllocation,
 ) (
-	string, error,
+	string, bool, error,
 ) {
+	var (
+		thisHost = hostBootstrap.ThisHost()
+		id       = daecommon.BootstrapGarageHostForAlloc(thisHost, alloc).ID
 
-	thisHost := hostBootstrap.ThisHost()
-	id := daecommon.BootstrapGarageHostForAlloc(thisHost, alloc).ID
-
-	peer := garage.LocalPeer{
+		peer = garage.LocalPeer{
 			RemotePeer: garage.RemotePeer{
 				ID:        id,
 				IP:        thisHost.IP().String(),
@@ -82,11 +86,16 @@ func garageWriteChildConfig(
 			AdminPort: alloc.AdminPort,
 		}
 
-	garageTomlPath := filepath.Join(
+		garageTomlPath = filepath.Join(
 			runtimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
 		)
+	)
 
-	err := garagesrv.WriteGarageTomlFile(garageTomlPath, garagesrv.GarageTomlData{
+	changed, err := garagesrv.WriteGarageTomlFile(
+		ctx,
+		logger,
+		garageTomlPath,
+		garagesrv.GarageTomlData{
 			MetaPath: alloc.MetaPath,
 			DataPath: alloc.DataPath,
 
@@ -95,13 +104,20 @@ func garageWriteChildConfig(
 
 			LocalPeer:      peer,
 			BootstrapPeers: hostBootstrap.GaragePeers(),
-	})
+		},
+	)
 
 	if err != nil {
-		return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
+		return "", false, fmt.Errorf(
+			"creating garage.toml file at %q: %w", garageTomlPath, err,
+		)
 	}
 
-	return garageTomlPath, nil
+	return garageTomlPath, changed, nil
+}
+
+func garagePmuxProcName(alloc daecommon.ConfigStorageAllocation) string {
+	return fmt.Sprintf("garage-%d", alloc.RPCPort)
 }
 
 func garagePmuxProcConfigs(
@@ -120,20 +136,22 @@ func garagePmuxProcConfigs(
 	)
 
 	if len(allocs) > 0 && rpcSecret == "" {
-		logger.WarnString(ctx, "Not starting garage instances for storage allocations, missing garage RPC secret")
-		return nil, nil
+		return nil, errors.New("Storage allocations defined, but garage RPC secret is not available")
 	}
 
 	for _, alloc := range allocs {
-
-		childConfigPath, err := garageWriteChildConfig(
-			rpcSecret, runtimeDirPath, adminToken, hostBootstrap, alloc,
+		childConfigPath, _, err := garageWriteChildConfig(
+			ctx,
+			logger,
+			rpcSecret, runtimeDirPath, adminToken,
+			hostBootstrap,
+			alloc,
 		)
 		if err != nil {
 			return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
 		}
 
-		procName := fmt.Sprintf("garage-%d", alloc.RPCPort)
+		procName := garagePmuxProcName(alloc)
 		pmuxProcConfigs[procName] = pmuxlib.ProcessConfig{
 			Cmd:  filepath.Join(binDirPath, "garage"),
 			Args: []string{"-c", childConfigPath, "server"},

go/daemon/children/nebula.go

@@ -3,9 +3,10 @@ package children
 import (
 	"context"
 	"fmt"
+	"io"
 	"isle/bootstrap"
 	"isle/daemon/daecommon"
-	"isle/yamlutil"
+	"isle/toolkit"
 	"net"
 	"path/filepath"
 
@@ -13,6 +14,7 @@ import (
 	"dev.mediocregopher.com/mediocre-go-lib.git/mctx"
 	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
 	"github.com/slackhq/nebula/cert"
+	"gopkg.in/yaml.v3"
 )
 
 // waitForNebula waits for the nebula interface to have been started up. It does
@@ -33,7 +35,7 @@ func waitForNebula(
 	until(
 		ctx,
 		logger,
-		"Creating UDP socket from nebula addr",
+		"Checking if nebula is online by creating UDP socket from nebula IP",
 		func(context.Context) error {
 			conn, err := net.DialUDP("udp", lUDPAddr, rUDPAddr)
 			if err != nil {
@@ -47,6 +49,7 @@ func waitForNebula(
 	return ctx.Err()
 }
 
+// TODO this needs to be produce a deterministic config value.
 func nebulaConfig(
 	networkConfig daecommon.NetworkConfig,
 	hostBootstrap bootstrap.Bootstrap,
@@ -113,16 +116,22 @@ func nebulaConfig(
 
 	} else {
 
-		_, port, err := net.SplitHostPort(publicAddr)
-
+		host, port, err := net.SplitHostPort(publicAddr)
 		if err != nil {
 			return nil, fmt.Errorf(
 				"parsing public address %q: %w", publicAddr, err,
 			)
 		}
 
+		// This helps with integration testing, so we can set a test to listen
+		// on some local IP without conflicting with something else running on
+		// the host.
+		if hostIP := net.ParseIP(host); hostIP == nil || !hostIP.IsLoopback() {
+			host = "0.0.0.0"
+		}
+
 		config["listen"] = map[string]string{
-			"host": "0.0.0.0",
+			"host": host,
 			"port": port,
 		}
 
@@ -136,44 +145,50 @@ func nebulaConfig(
 }
 
 func nebulaWriteConfig(
+	ctx context.Context,
+	logger *mlog.Logger,
 	runtimeDirPath string,
 	networkConfig daecommon.NetworkConfig,
 	hostBootstrap bootstrap.Bootstrap,
 ) (
-	string, error,
+	string, bool, error,
 ) {
 	config, err := nebulaConfig(networkConfig, hostBootstrap)
 	if err != nil {
-		return "", fmt.Errorf("creating nebula config: %w", err)
+		return "", false, fmt.Errorf("creating nebula config: %w", err)
 	}
 
 	nebulaYmlPath := filepath.Join(runtimeDirPath, "nebula.yml")
 
-	if err := yamlutil.WriteYamlFile(config, nebulaYmlPath, 0600); err != nil {
-		return "", fmt.Errorf("writing nebula.yml to %q: %w", nebulaYmlPath, err)
+	changed, err := toolkit.WriteFileCheckChanged(
+		ctx, logger, nebulaYmlPath, 0600, func(w io.Writer) error {
+			return yaml.NewEncoder(w).Encode(config)
+		},
+	)
+	if err != nil {
+		return "", false, fmt.Errorf(
+			"writing nebula.yml to %q: %w", nebulaYmlPath, err,
+		)
 	}
 
-	return nebulaYmlPath, nil
+	return nebulaYmlPath, changed, nil
 }
 
 func nebulaPmuxProcConfig(
+	ctx context.Context,
+	logger *mlog.Logger,
 	runtimeDirPath, binDirPath string,
 	networkConfig daecommon.NetworkConfig,
 	hostBootstrap bootstrap.Bootstrap,
 ) (
 	pmuxlib.ProcessConfig, error,
 ) {
-	config, err := nebulaConfig(networkConfig, hostBootstrap)
+	nebulaYmlPath, _, err := nebulaWriteConfig(
+		ctx, logger, runtimeDirPath, networkConfig, hostBootstrap,
+	)
 	if err != nil {
 		return pmuxlib.ProcessConfig{}, fmt.Errorf(
-			"creating nebula config: %w", err,
-		)
-	}
-
-	nebulaYmlPath := filepath.Join(runtimeDirPath, "nebula.yml")
-	if err := yamlutil.WriteYamlFile(config, nebulaYmlPath, 0600); err != nil {
-		return pmuxlib.ProcessConfig{}, fmt.Errorf(
-			"writing nebula.yml to %q: %w", nebulaYmlPath, err,
+			"writing nebula config: %w", err,
 		)
 	}
 

go/daemon/children/pmux.go

@@ -19,6 +19,8 @@ func (c *Children) newPmuxConfig(
 	pmuxlib.Config, error,
 ) {
 	nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(
+		ctx,
+		c.logger,
 		c.runtimeDir.Path,
 		binDirPath,
 		networkConfig,
@@ -29,6 +31,7 @@ func (c *Children) newPmuxConfig(
 	}
 
 	dnsmasqPmuxProcConfig, err := dnsmasqPmuxProcConfig(
+		ctx,
 		c.logger,
 		c.runtimeDir.Path,
 		binDirPath,
@@ -72,12 +75,10 @@ func (c *Children) postPmuxInit(
 	garageAdminToken string,
 	hostBootstrap bootstrap.Bootstrap,
 ) error {
-	c.logger.Info(ctx, "Waiting for nebula VPN to come online")
 	if err := waitForNebula(ctx, c.logger, hostBootstrap); err != nil {
 		return fmt.Errorf("waiting for nebula to start: %w", err)
 	}
 
-	c.logger.Info(ctx, "Waiting for garage instances to come online")
 	err := waitForGarage(
 		ctx, c.logger, networkConfig, garageAdminToken, hostBootstrap,
 	)

go/daemon/client.go

@@ -9,6 +9,7 @@ package daemon
 import (
 	"context"
 	"isle/bootstrap"
+	"isle/daemon/daecommon"
 	"isle/daemon/jsonrpc2"
 	"isle/daemon/network"
 	"isle/nebula"
@@ -59,6 +60,15 @@ func (c *rpcClient) CreateNetwork(ctx context.Context, name string, domain strin
 	return
 }
 
+func (c *rpcClient) GetConfig(ctx context.Context) (n1 daecommon.NetworkConfig, err error) {
+	err = c.client.Call(
+		ctx,
+		&n1,
+		"GetConfig",
+	)
+	return
+}
+
 func (c *rpcClient) GetGarageClientParams(ctx context.Context) (g1 network.GarageClientParams, err error) {
 	err = c.client.Call(
 		ctx,
@@ -114,3 +124,13 @@ func (c *rpcClient) RemoveHost(ctx context.Context, hostName nebula.HostName) (e
 	)
 	return
 }
+
+func (c *rpcClient) SetConfig(ctx context.Context, n1 daecommon.NetworkConfig) (err error) {
+	err = c.client.Call(
+		ctx,
+		nil,
+		"SetConfig",
+		n1,
+	)
+	return
+}

go/daemon/daecommon/config.go

@@ -6,8 +6,8 @@ import (
 	"io"
 	"isle/bootstrap"
 	"isle/toolkit"
+	"isle/yamlutil"
 	"net"
-	"os"
 	"strconv"
 
 	_ "embed"
@@ -71,7 +71,7 @@ type NetworkConfig struct {
 		Tun        ConfigTun      `yaml:"tun"`
 	} `yaml:"vpn"`
 	Storage struct {
-		Allocations []ConfigStorageAllocation
+		Allocations []ConfigStorageAllocation `yaml:"allocations"`
 	} `yaml:"storage"`
 }
 
@@ -145,6 +145,18 @@ func (c *NetworkConfig) fillDefaults() {
 	)
 }
 
+// UnmarshalYAML implements the yaml.Unmarshaler interface. It will attempt to
+// fill in default values where it can.
+func (c *NetworkConfig) UnmarshalYAML(n *yaml.Node) error {
+	type wrap NetworkConfig
+	if err := n.Decode((*wrap)(c)); err != nil {
+		return fmt.Errorf("decoding into %T: %w", c, err)
+	}
+
+	c.fillDefaults()
+	return nil
+}
+
 // Config describes the structure of the daemon config file.
 type Config struct {
 	Networks map[string]NetworkConfig `yaml:"networks"`
@@ -186,6 +198,30 @@ func CopyDefaultConfig(into io.Writer) error {
 	return err
 }
 
+// UnmarshalYAML implements the yaml.Unmarshaler interface. It will attempt to
+// fill in default values where it can.
+func (c *Config) UnmarshalYAML(n *yaml.Node) error {
+	{ // DEPRECATED
+		var networkConfig NetworkConfig
+		_ = n.Decode(&networkConfig)
+		if !toolkit.IsZero(networkConfig) {
+			*c = Config{
+				Networks: map[string]NetworkConfig{
+					DeprecatedNetworkID: networkConfig,
+				},
+			}
+			return c.Validate()
+		}
+	}
+
+	type wrap Config
+	if err := n.Decode((*wrap)(c)); err != nil {
+		return fmt.Errorf("yaml unmarshaling back into Config struct: %w", err)
+	}
+
+	return c.Validate()
+}
+
 // LoadConfig loads the daemon config from userConfigPath.
 //
 // If userConfigPath is not given then the default is loaded and returned.
@@ -194,47 +230,17 @@ func LoadConfig(userConfigPath string) (Config, error) {
 		return Config{}, nil
 	}
 
-	userConfigB, err := os.ReadFile(userConfigPath)
-	if err != nil {
-		return Config{}, fmt.Errorf("reading from file: %w", err)
-	}
-
-	{ // DEPRECATED
-		var networkConfig NetworkConfig
-		_ = yaml.Unmarshal(userConfigB, &networkConfig)
-		if !toolkit.IsZero(networkConfig) {
-			networkConfig.fillDefaults()
-			config := Config{
-				Networks: map[string]NetworkConfig{
-					DeprecatedNetworkID: networkConfig,
-				},
-			}
-			return config, config.Validate()
-		}
-	}
-
 	var config Config
-	if err := yaml.Unmarshal(userConfigB, &config); err != nil {
-		return Config{}, fmt.Errorf("yaml unmarshaling back into Config struct: %w", err)
-	}
-
-	for id := range config.Networks {
-		network := config.Networks[id]
-		network.fillDefaults()
-		config.Networks[id] = network
-	}
-
-	return config, config.Validate()
+	err := yamlutil.LoadYamlFile(&config, userConfigPath)
+	return config, err
 }
 
 // BootstrapGarageHostForAlloc returns the bootstrap.GarageHostInstance which
 // corresponds with the given alloc from the daemon config. This will panic if
 // no associated instance can be found.
 func BootstrapGarageHostForAlloc(
-	host bootstrap.Host,
-	alloc ConfigStorageAllocation,
+	host bootstrap.Host, alloc ConfigStorageAllocation,
 ) bootstrap.GarageHostInstance {
-
 	for _, inst := range host.Garage.Instances {
 		if inst.RPCPort == alloc.RPCPort {
 			return inst

go/daemon/daemon.go

@@ -138,7 +138,6 @@ func New(
 		d.networks[id], err = network.Load(
 			ctx,
 			logger.WithNamespace("network"),
-			id,
 			networkConfig,
 			d.envBinDirPath,
 			networkStateDir,
@@ -440,6 +439,39 @@ func (d *Daemon) CreateNebulaCertificate(
 	)
 }
 
+func (d *Daemon) GetConfig(
+	ctx context.Context,
+) (
+	daecommon.NetworkConfig, error,
+) {
+	return withNetwork(
+		ctx,
+		d,
+		func(
+			ctx context.Context, n network.Network,
+		) (
+			daecommon.NetworkConfig, error,
+		) {
+			return n.GetConfig(ctx)
+		},
+	)
+}
+
+func (d *Daemon) SetConfig(
+	ctx context.Context, config daecommon.NetworkConfig,
+) error {
+	_, err := withNetwork(
+		ctx,
+		d,
+		func(ctx context.Context, n network.Network) (struct{}, error) {
+			// TODO needs to check that public addresses aren't being shared
+			// across networks, and whatever else happens in Config.Validate.
+			return struct{}{}, n.SetConfig(ctx, config)
+		},
+	)
+	return err
+}
+
 // Shutdown blocks until all resources held or created by the daemon,
 // including child processes it has started, have been cleaned up.
 //

go/daemon/jsonrpc2/jsonrpc2_test.go

@@ -70,12 +70,9 @@ type divider interface {
 	Divide2FromMeta(ctx context.Context) (int, error)
 }
 
-var testHandler = func() Handler {
+func testHandler(t *testing.T) Handler {
 	var (
-		logger = mlog.NewLogger(&mlog.LoggerOpts{
-			MaxLevel: mlog.LevelDebug.Int(),
-		})
-
+		logger = mlog.NewTestLogger(t)
 		d      = divider(dividerImpl{})
 	)
 
@@ -85,7 +82,7 @@ var testHandler = func() Handler {
 	)(
 		NewDispatchHandler(&d),
 	)
-}()
+}
 
 func testClient(t *testing.T, client Client) {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -194,7 +191,7 @@ func TestReadWriter(t *testing.T) {
 		clientRW  = rw{clientReader, clientWriter}
 		handlerRW = rw{handlerReader, handlerWriter}
 
-		server = NewReadWriterServer(testHandler, handlerRW)
+		server = NewReadWriterServer(testHandler(t), handlerRW)
 		client = NewReadWriterClient(clientRW)
 
 		wg = new(sync.WaitGroup)
@@ -220,7 +217,7 @@ func TestReadWriter(t *testing.T) {
 }
 
 func TestHTTP(t *testing.T) {
-	server := httptest.NewServer(NewHTTPHandler(testHandler))
+	server := httptest.NewServer(NewHTTPHandler(testHandler(t)))
 	t.Cleanup(server.Close)
 	testClient(t, NewHTTPClient(server.URL))
 }
@@ -228,7 +225,7 @@ func TestHTTP(t *testing.T) {
 func TestUnixHTTP(t *testing.T) {
 	var (
 		unixSocketPath = filepath.Join(t.TempDir(), "test.sock")
-		server         = httptest.NewUnstartedServer(NewHTTPHandler(testHandler))
+		server         = httptest.NewUnstartedServer(NewHTTPHandler(testHandler(t)))
 	)
 
 	var err error

go/daemon/network/garage.go

@@ -25,19 +25,21 @@ const (
 	garageGlobalBucketBootstrapHostsDirPath = "bootstrap/hosts"
 )
 
-func (n *network) getGarageClientParams(
-	ctx context.Context, currBootstrap bootstrap.Bootstrap,
+func getGarageClientParams(
+	ctx context.Context,
+	secretsStore secrets.Store,
+	currBootstrap bootstrap.Bootstrap,
 ) (
 	GarageClientParams, error,
 ) {
 	creds, err := daecommon.GetGarageS3APIGlobalBucketCredentials(
-		ctx, n.secretsStore,
+		ctx, secretsStore,
 	)
 	if err != nil {
 		return GarageClientParams{}, fmt.Errorf("getting garage global bucket creds: %w", err)
 	}
 
-	rpcSecret, err := daecommon.GetGarageRPCSecret(ctx, n.secretsStore)
+	rpcSecret, err := daecommon.GetGarageRPCSecret(ctx, secretsStore)
 	if err != nil && !errors.Is(err, secrets.ErrNotFound) {
 		return GarageClientParams{}, fmt.Errorf("getting garage rpc secret: %w", err)
 	}
@@ -92,6 +94,8 @@ func garageApplyLayout(
 		peers    = make([]garage.PeerLayout, len(allocs))
 	)
 
+	defer adminClient.Close()
+
 	for i, alloc := range allocs {
 
 		id := daecommon.BootstrapGarageHostForAlloc(thisHost, alloc).ID
@@ -124,6 +128,7 @@ func garageInitializeGlobalBucket(
 	adminClient := newGarageAdminClient(
 		logger, networkConfig, adminToken, hostBootstrap,
 	)
+	defer adminClient.Close()
 
 	creds, err := adminClient.CreateS3APICredentials(
 		ctx, garage.GlobalBucketS3APICredentialsName,
@@ -152,12 +157,17 @@ func garageInitializeGlobalBucket(
 	return creds, nil
 }
 
-func (n *network) getGarageBootstrapHosts(
-	ctx context.Context, currBootstrap bootstrap.Bootstrap,
+func getGarageBootstrapHosts(
+	ctx context.Context,
+	logger *mlog.Logger,
+	secretsStore secrets.Store,
+	currBootstrap bootstrap.Bootstrap,
 ) (
 	map[nebula.HostName]bootstrap.Host, error,
 ) {
-	garageClientParams, err := n.getGarageClientParams(ctx, currBootstrap)
+	garageClientParams, err := getGarageClientParams(
+		ctx, secretsStore, currBootstrap,
+	)
 	if err != nil {
 		return nil, fmt.Errorf("getting garage client params: %w", err)
 	}
@@ -175,6 +185,8 @@ func (n *network) getGarageBootstrapHosts(
 		)
 	)
 
+	defer client.Close()
+
 	for objInfo := range objInfoCh {
 
 		ctx := mctx.Annotate(ctx, "objectKey", objInfo.Key)
@@ -197,13 +209,13 @@ func (n *network) getGarageBootstrapHosts(
 		obj.Close()
 
 		if err != nil {
-			n.logger.Warn(ctx, "Object contains invalid json", err)
+			logger.Warn(ctx, "Object contains invalid json", err)
 			continue
 		}
 
 		host, err := authedHost.Unwrap(currBootstrap.CAPublicCredentials)
 		if err != nil {
-			n.logger.Warn(ctx, "Host could not be authenticated", err)
+			logger.Warn(ctx, "Host could not be authenticated", err)
 		}
 
 		hosts[host.Name] = host
@@ -215,10 +227,14 @@ func (n *network) getGarageBootstrapHosts(
 // putGarageBoostrapHost places the <hostname>.json.signed file for this host
 // into garage so that other hosts are able to see relevant configuration for
 // it.
-func (n *network) putGarageBoostrapHost(
-	ctx context.Context, currBootstrap bootstrap.Bootstrap,
+func putGarageBoostrapHost(
+	ctx context.Context,
+	secretsStore secrets.Store,
+	currBootstrap bootstrap.Bootstrap,
 ) error {
-	garageClientParams, err := n.getGarageClientParams(ctx, currBootstrap)
+	garageClientParams, err := getGarageClientParams(
+		ctx, secretsStore, currBootstrap,
+	)
 	if err != nil {
 		return fmt.Errorf("getting garage client params: %w", err)
 	}
@@ -228,6 +244,8 @@ func (n *network) putGarageBoostrapHost(
 		client = garageClientParams.GlobalBucketS3APIClient()
 	)
 
+	defer client.Close()
+
 	configured, err := nebula.Sign(
 		host.HostConfigured, currBootstrap.PrivateCredentials.SigningPrivateKey,
 	)
@@ -265,7 +283,7 @@ func (n *network) putGarageBoostrapHost(
 }
 
 func removeGarageBootstrapHost(
-	ctx context.Context, client garage.S3APIClient, hostName nebula.HostName,
+	ctx context.Context, client *garage.S3APIClient, hostName nebula.HostName,
 ) error {
 
 	filePath := filepath.Join(

go/daemon/network/network.go

@@ -39,7 +39,7 @@ type GarageClientParams struct {
 
 // GlobalBucketS3APIClient returns an S3 client pre-configured with access to
 // the global bucket.
-func (p GarageClientParams) GlobalBucketS3APIClient() garage.S3APIClient {
+func (p GarageClientParams) GlobalBucketS3APIClient() *garage.S3APIClient {
 	var (
 		addr  = p.Peer.S3APIAddr()
 		creds = p.GlobalBucketS3APICredentials
@@ -110,6 +110,13 @@ type RPC interface {
 	) (
 		nebula.Certificate, error,
 	)
+
+	// GetConfig returns the configuration currently in use.
+	GetConfig(context.Context) (daecommon.NetworkConfig, error)
+
+	// SetConfig overrides the current config with the given one, adjusting any
+	// running child processes as needed.
+	SetConfig(context.Context, daecommon.NetworkConfig) error
 }
 
 // Network manages membership in a single micropelago network. Each Network
@@ -142,18 +149,24 @@ type Network interface {
 // Network instance. A nil Opts is equivalent to a zero value.
 type Opts struct {
 	ChildrenOpts *children.Opts
+
+	GarageAdminToken string // Will be randomly generated if left unset.
 }
 
 func (o *Opts) withDefaults() *Opts {
 	if o == nil {
 		o = new(Opts)
 	}
+
+	if o.GarageAdminToken == "" {
+		o.GarageAdminToken = toolkit.RandStr(32)
+	}
+
 	return o
 }
 
 type network struct {
 	logger *mlog.Logger
-	networkConfig daecommon.NetworkConfig
 
 	envBinDirPath string
 	stateDir      toolkit.Dir
@@ -162,10 +175,10 @@ type network struct {
 	opts *Opts
 
 	secretsStore secrets.Store
-	garageAdminToken string
 
 	l             sync.RWMutex
 	children      *children.Children
+	networkConfig daecommon.NetworkConfig
 	currBootstrap bootstrap.Bootstrap
 
 	shutdownCh chan struct{}
@@ -176,7 +189,6 @@ type network struct {
 // been initialized.
 func instatiateNetwork(
 	logger *mlog.Logger,
-	networkID string,
 	networkConfig daecommon.NetworkConfig,
 	envBinDirPath string,
 	stateDir toolkit.Dir,
@@ -190,7 +202,6 @@ func instatiateNetwork(
 		stateDir:      stateDir,
 		runtimeDir:    runtimeDir,
 		opts:          opts.withDefaults(),
-		garageAdminToken: toolkit.RandStr(32),
 		shutdownCh:    make(chan struct{}),
 	}
 }
@@ -224,7 +235,6 @@ func LoadCreationParams(
 func Load(
 	ctx context.Context,
 	logger *mlog.Logger,
-	networkID string,
 	networkConfig daecommon.NetworkConfig,
 	envBinDirPath string,
 	stateDir toolkit.Dir,
@@ -235,7 +245,6 @@ func Load(
 ) {
 	n := instatiateNetwork(
 		logger,
-		networkID,
 		networkConfig,
 		envBinDirPath,
 		stateDir,
@@ -281,7 +290,6 @@ func Join(
 ) {
 	n := instatiateNetwork(
 		logger,
-		joiningBootstrap.Bootstrap.NetworkCreationParams.ID,
 		networkConfig,
 		envBinDirPath,
 		stateDir,
@@ -348,7 +356,6 @@ func Create(
 
 	n := instatiateNetwork(
 		logger,
-		creationParams.ID,
 		networkConfig,
 		envBinDirPath,
 		stateDir,
@@ -429,8 +436,8 @@ func (n *network) initialize(
 		n.secretsStore,
 		n.networkConfig,
 		n.runtimeDir,
-		n.garageAdminToken,
-		currBootstrap,
+		n.opts.GarageAdminToken,
+		n.currBootstrap,
 		n.opts.ChildrenOpts,
 	)
 	if err != nil {
@@ -445,8 +452,16 @@ func (n *network) initialize(
 		return fmt.Errorf("performing post-initialization: %w", err)
 	}
 
-	// TODO annotate this context with creation params
-	ctx, cancel := context.WithCancel(context.Background())
+	// Do this now so that everything is stable before returning. This also
+	// serves a dual-purpose, as it makes sure that the PUT from the postInit
+	// above has propagated from the local garage instance, if there is one.
+	n.logger.Info(ctx, "Reloading hosts from network storage")
+	if err = n.reloadHosts(ctx); err != nil {
+		return fmt.Errorf("Reloading network bootstrap: %w", err)
+	}
+
+	ctx = context.WithoutCancel(ctx)
+	ctx, cancel := context.WithCancel(ctx)
 	n.wg.Add(1)
 	go func() {
 		defer n.wg.Done()
@@ -458,7 +473,7 @@ func (n *network) initialize(
 	go func() {
 		defer n.wg.Done()
 		n.reloadLoop(ctx)
-		n.logger.Debug(ctx, "Daemon restart loop stopped")
+		n.logger.Debug(ctx, "Daemon reload loop stopped")
 	}()
 
 	return nil
@@ -468,7 +483,11 @@ func (n *network) postInit(ctx context.Context) error {
 	if len(n.networkConfig.Storage.Allocations) > 0 {
 		n.logger.Info(ctx, "Applying garage layout")
 		if err := garageApplyLayout(
-			ctx, n.logger, n.networkConfig, n.garageAdminToken, n.currBootstrap,
+			ctx,
+			n.logger,
+			n.networkConfig,
+			n.opts.GarageAdminToken,
+			n.currBootstrap,
 		); err != nil {
 			return fmt.Errorf("applying garage layout: %w", err)
 		}
@@ -488,7 +507,7 @@ func (n *network) postInit(ctx context.Context) error {
 			ctx,
 			n.logger,
 			n.networkConfig,
-			n.garageAdminToken,
+			n.opts.GarageAdminToken,
 			n.currBootstrap,
 		)
 		if err != nil {
@@ -504,7 +523,7 @@ func (n *network) postInit(ctx context.Context) error {
 	}
 
 	n.logger.Info(ctx, "Updating host info in garage")
-	err = n.putGarageBoostrapHost(ctx, n.currBootstrap)
+	err = putGarageBoostrapHost(ctx, n.secretsStore, n.currBootstrap)
 	if err != nil {
 		return fmt.Errorf("updating host info in garage: %w", err)
 	}
@@ -512,8 +531,39 @@ func (n *network) postInit(ctx context.Context) error {
 	return nil
 }
 
+func (n *network) reloadHosts(ctx context.Context) error {
+	n.l.RLock()
+	networkConfig := n.networkConfig
+	currBootstrap := n.currBootstrap
+	n.l.RUnlock()
+
+	n.logger.Info(ctx, "Checking for bootstrap changes")
+	newHosts, err := getGarageBootstrapHosts(
+		ctx, n.logger, n.secretsStore, currBootstrap,
+	)
+	if err != nil {
+		return fmt.Errorf("getting hosts from garage: %w", err)
+	}
+
+	// TODO there's some potential race conditions here, where
+	// CreateHost could be called at this point, write the new host to
+	// garage and the bootstrap, but then this reload call removes the
+	// host from this bootstrap/children until the next reload.
+
+	newBootstrap := currBootstrap
+	newBootstrap.Hosts = newHosts
+
+	err = n.reload(ctx, networkConfig, newBootstrap)
+	if err != nil {
+		return fmt.Errorf("reloading with new host data: %w", err)
+	}
+
+	return nil
+}
+
 func (n *network) reloadLoop(ctx context.Context) {
-	ticker := time.NewTicker(3 * time.Minute)
+	const period = 3 * time.Minute
+	ticker := time.NewTicker(period)
 	defer ticker.Stop()
 
 	for {
@@ -522,66 +572,44 @@ func (n *network) reloadLoop(ctx context.Context) {
 			return
 
 		case <-ticker.C:
-			n.l.RLock()
-			currBootstrap := n.currBootstrap
-			n.l.RUnlock()
-
-			n.logger.Info(ctx, "Checking for bootstrap changes")
-			newHosts, err := n.getGarageBootstrapHosts(ctx, currBootstrap)
-			if err != nil {
-				n.logger.Error(ctx, "Failed to get hosts from garage", err)
-				continue
-			}
-
-			// TODO there's some potential race conditions here, where
-			// CreateHost could be called at this point, write the new host to
-			// garage and the bootstrap, but then this reload call removes the
-			// host from this bootstrap/children until the next reload.
-
-			if err := n.reload(ctx, currBootstrap, newHosts); err != nil {
-				n.logger.Error(ctx, "Reloading with new host data failed", err)
+			if err := n.reloadHosts(ctx); err != nil {
+				n.logger.Error(ctx, "Attempting to reload", err)
 				continue
 			}
 		}
 	}
 }
 
-// reload will check the existing hosts data from currBootstrap against a
-// potentially updated set of hosts data, and if there are any differences will
-// perform whatever changes are necessary.
+// reload will check the existing hosts data from currBootstrap against
+// a potentially updated set of hosts data, and if there are any differences
+// will perform whatever changes are necessary.
 func (n *network) reload(
 	ctx context.Context,
-	currBootstrap bootstrap.Bootstrap,
-	newHosts map[nebula.HostName]bootstrap.Host,
+	newNetworkConfig daecommon.NetworkConfig,
+	newBootstrap bootstrap.Bootstrap,
 ) error {
-	var (
-		newBootstrap = currBootstrap
-		thisHost     = currBootstrap.ThisHost()
-	)
-
-	newBootstrap.Hosts = newHosts
+	n.l.Lock()
+	defer n.l.Unlock()
 
 	// the daemon's view of this host's bootstrap info takes precedence over
-	// whatever is in garage
+	// whatever is in garage. The garage version lacks the private credentials
+	// which must be stored locally.
+	thisHost := n.currBootstrap.ThisHost()
 	newBootstrap.Hosts[thisHost.Name] = thisHost
 
-	diff, err := children.CalculateReloadDiff(
-		n.networkConfig, currBootstrap, newBootstrap,
-	)
+	n.logger.Info(ctx, "Writing updated bootstrap to state dir")
+	err := writeBootstrapToStateDir(n.stateDir.Path, newBootstrap)
 	if err != nil {
-		return fmt.Errorf("calculating diff between bootstraps: %w", err)
-	} else if diff == (children.ReloadDiff{}) {
-		n.logger.Info(ctx, "No changes to bootstrap detected")
-		return nil
+		return fmt.Errorf("writing bootstrap to state dir: %w", err)
 	}
 
-	n.logger.Info(ctx, "Bootstrap has changed, storing new bootstrap")
-	n.l.Lock()
+	n.networkConfig = newNetworkConfig
 	n.currBootstrap = newBootstrap
-	n.l.Unlock()
 
-	if err := n.children.Reload(ctx, newBootstrap, diff); err != nil {
-		return fmt.Errorf("reloading child processes (diff:%+v): %w", diff, err)
+	n.logger.Info(ctx, "Reloading child processes")
+	err = n.children.Reload(ctx, newNetworkConfig, newBootstrap)
+	if err != nil {
+		return fmt.Errorf("reloading child processes: %w", err)
 	}
 
 	return nil
@@ -597,9 +625,7 @@ func withCurrBootstrap[Res any](
 	return fn(currBootstrap)
 }
 
-func (n *network) getBootstrap(
-	ctx context.Context,
-) (
+func (n *network) getBootstrap() (
 	bootstrap.Bootstrap, error,
 ) {
 	return withCurrBootstrap(n, func(
@@ -612,17 +638,17 @@ func (n *network) getBootstrap(
 }
 
 func (n *network) GetHosts(ctx context.Context) ([]bootstrap.Host, error) {
-	b, err := n.getBootstrap(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("retrieving bootstrap: %w", err)
-	}
-
-	hosts := maps.Values(b.Hosts)
+	return withCurrBootstrap(n, func(
+		currBootstrap bootstrap.Bootstrap,
+	) (
+		[]bootstrap.Host, error,
+	) {
+		hosts := maps.Values(currBootstrap.Hosts)
 		slices.SortFunc(hosts, func(a, b bootstrap.Host) int {
 			return cmp.Compare(a.Name, b.Name)
 		})
-
 		return hosts, nil
+	})
 }
 
 func (n *network) GetGarageClientParams(
@@ -635,7 +661,7 @@ func (n *network) GetGarageClientParams(
 	) (
 		GarageClientParams, error,
 	) {
-		return n.getGarageClientParams(ctx, currBootstrap)
+		return getGarageClientParams(ctx, n.secretsStore, currBootstrap)
 	})
 }
 
@@ -644,7 +670,7 @@ func (n *network) GetNebulaCAPublicCredentials(
 ) (
 	nebula.CAPublicCredentials, error,
 ) {
-	b, err := n.getBootstrap(ctx)
+	b, err := n.getBootstrap()
 	if err != nil {
 		return nebula.CAPublicCredentials{}, fmt.Errorf(
 			"retrieving bootstrap: %w", err,
@@ -662,12 +688,16 @@ func (n *network) RemoveHost(ctx context.Context, hostName nebula.HostName) erro
 	) (
 		struct{}, error,
 	) {
-		garageClientParams, err := n.getGarageClientParams(ctx, currBootstrap)
+		garageClientParams, err := getGarageClientParams(
+			ctx, n.secretsStore, currBootstrap,
+		)
 		if err != nil {
 			return struct{}{}, fmt.Errorf("get garage client params: %w", err)
 		}
 
 		client := garageClientParams.GlobalBucketS3APIClient()
+		defer client.Close()
+
 		return struct{}{}, removeGarageBootstrapHost(ctx, client, hostName)
 	})
 	return err
@@ -770,6 +800,7 @@ func (n *network) CreateHost(
 	JoiningBootstrap, error,
 ) {
 	n.l.RLock()
+	networkConfig := n.networkConfig
 	currBootstrap := n.currBootstrap
 	n.l.RUnlock()
 
@@ -823,17 +854,19 @@ func (n *network) CreateHost(
 	}
 
 	n.logger.Info(ctx, "Putting new host in garage")
-	err = n.putGarageBoostrapHost(ctx, joiningBootstrap.Bootstrap)
+	err = putGarageBoostrapHost(ctx, n.secretsStore, joiningBootstrap.Bootstrap)
 	if err != nil {
 		return JoiningBootstrap{}, fmt.Errorf("putting new host in garage: %w", err)
 	}
 
 	// the new bootstrap will have been initialized with both all existing hosts
 	// (based on currBootstrap) and the host being created.
-	newHosts := joiningBootstrap.Bootstrap.Hosts
+	newBootstrap := currBootstrap
+	newBootstrap.Hosts = joiningBootstrap.Bootstrap.Hosts
 
 	n.logger.Info(ctx, "Reloading local state with new host")
-	if err := n.reload(ctx, currBootstrap, newHosts); err != nil {
+	err = n.reload(ctx, networkConfig, newBootstrap)
+	if err != nil {
 		return JoiningBootstrap{}, fmt.Errorf("reloading child processes: %w", err)
 	}
 
@@ -871,12 +904,66 @@ func (n *network) CreateNebulaCertificate(
 	})
 }
 
+func (n *network) GetConfig(context.Context) (daecommon.NetworkConfig, error) {
+	n.l.RLock()
+	defer n.l.RUnlock()
+	return n.networkConfig, nil
+}
+
+func (n *network) SetConfig(
+	ctx context.Context, config daecommon.NetworkConfig,
+) error {
+	newBootstrap, err := coalesceNetworkConfigAndBootstrap(
+		config, n.currBootstrap,
+	)
+	if err != nil {
+		return fmt.Errorf("combining configuration into bootstrap: %w", err)
+	}
+
+	n.l.Lock()
+	defer n.l.Unlock()
+
+	n.logger.Info(ctx, "Shutting down children")
+	n.children.Shutdown()
+
+	err = writeBootstrapToStateDir(n.stateDir.Path, newBootstrap)
+	if err != nil {
+		return fmt.Errorf("writing bootstrap to state dir: %w", err)
+	}
+
+	n.networkConfig = config
+	n.currBootstrap = newBootstrap
+
+	n.logger.Info(ctx, "Creating child processes")
+	n.children, err = children.New(
+		ctx,
+		n.logger.WithNamespace("children"),
+		n.envBinDirPath,
+		n.secretsStore,
+		n.networkConfig,
+		n.runtimeDir,
+		n.opts.GarageAdminToken,
+		n.currBootstrap,
+		n.opts.ChildrenOpts,
+	)
+	if err != nil {
+		return fmt.Errorf("creating child processes: %w", err)
+	}
+
+	n.logger.Info(ctx, "Child processes re-created")
+
+	if err := n.postInit(ctx); err != nil {
+		return fmt.Errorf("performing post-initialization: %w", err)
+	}
+
+	return nil
+}
+
 func (n *network) GetNetworkCreationParams(
 	ctx context.Context,
 ) (
 	bootstrap.CreationParams, error,
 ) {
-
 	return withCurrBootstrap(n, func(
 		currBootstrap bootstrap.Bootstrap,
 	) (

go/daemon/network/network_it_test.go

@@ -0,0 +1,156 @@
+package network
+
+import (
+	"isle/bootstrap"
+	"isle/daemon/daecommon"
+	"isle/garage"
+	"isle/jsonutil"
+	"isle/nebula"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCreate(t *testing.T) {
+	var (
+		h       = newIntegrationHarness(t)
+		network = h.createNetwork(t, "primus", nil)
+	)
+
+	gotCreationParams, err := LoadCreationParams(network.stateDir)
+	assert.NoError(t, err)
+	assert.Equal(t, gotCreationParams, network.creationParams)
+}
+
+func TestLoad(t *testing.T) {
+	var (
+		h       = newIntegrationHarness(t)
+		network = h.createNetwork(t, "primus", &createNetworkOpts{
+			manualShutdown: true,
+		})
+	)
+
+	t.Log("Shutting down network")
+	assert.NoError(t, network.Shutdown())
+
+	t.Log("Calling Load")
+	loadedNetwork, err := Load(
+		h.ctx,
+		h.logger.WithNamespace("loadedNetwork"),
+		network.networkConfig,
+		getEnvBinDirPath(),
+		network.stateDir,
+		h.mkDir(t, "runtime"),
+		network.opts,
+	)
+	assert.NoError(t, err)
+
+	t.Cleanup(func() {
+		t.Log("Shutting down loadedNetwork")
+		assert.NoError(t, loadedNetwork.Shutdown())
+	})
+}
+
+func TestJoin(t *testing.T) {
+	var (
+		h        = newIntegrationHarness(t)
+		primus   = h.createNetwork(t, "primus", nil)
+		secondus = h.joinNetwork(t, primus, "secondus", nil)
+	)
+
+	primusHosts, err := primus.GetHosts(h.ctx)
+	assert.NoError(t, err)
+
+	secondusHosts, err := secondus.GetHosts(h.ctx)
+	assert.NoError(t, err)
+
+	assert.Equal(t, primusHosts, secondusHosts)
+}
+
+func TestNetwork_GetConfig(t *testing.T) {
+	var (
+		h       = newIntegrationHarness(t)
+		network = h.createNetwork(t, "primus", nil)
+	)
+
+	config, err := network.GetConfig(h.ctx)
+	assert.NoError(t, err)
+
+	assert.Equal(t, config, network.networkConfig)
+}
+
+func TestNetwork_SetConfig(t *testing.T) {
+	t.Run("adding storage alloc", func(t *testing.T) {
+		var (
+			h       = newIntegrationHarness(t)
+			network = h.createNetwork(t, "primus", nil)
+		)
+
+		network.networkConfig.Storage.Allocations = append(
+			network.networkConfig.Storage.Allocations,
+			daecommon.ConfigStorageAllocation{
+				DataPath:  h.mkDir(t, "data").Path,
+				MetaPath:  h.mkDir(t, "meta").Path,
+				Capacity:  1,
+				S3APIPort: 4900,
+				RPCPort:   4901,
+				AdminPort: 4902,
+			},
+		)
+
+		assert.NoError(t, network.SetConfig(h.ctx, network.networkConfig))
+
+		// Check that the Host information was updated
+		newHosts, err := network.GetHosts(h.ctx)
+		assert.NoError(t, err)
+
+		newHostsByName := map[nebula.HostName]bootstrap.Host{}
+		for _, h := range newHosts {
+			newHostsByName[h.Name] = h
+		}
+
+		newHost, ok := newHostsByName[network.hostName]
+		assert.True(t, ok)
+
+		allocs := newHost.HostConfigured.Garage.Instances
+		assert.Len(t, allocs, 4)
+
+		newAlloc := allocs[3]
+		assert.NotEmpty(t, newAlloc.ID)
+		newAlloc.ID = ""
+		assert.Equal(t, bootstrap.GarageHostInstance{
+			S3APIPort: 4900,
+			RPCPort:   4901,
+		}, newAlloc)
+
+		// Check that the bootstrap file was written with the new host config
+		var storedBootstrap bootstrap.Bootstrap
+		assert.NoError(t, jsonutil.LoadFile(
+			&storedBootstrap, bootstrap.StateDirPath(network.stateDir.Path),
+		))
+		assert.Equal(t, newHostsByName, storedBootstrap.Hosts)
+
+		// Check that garage layout contains the new allocation
+		garageAdminClient := newGarageAdminClient(
+			h.logger,
+			network.networkConfig,
+			network.opts.GarageAdminToken,
+			storedBootstrap,
+		)
+
+		layout, err := garageAdminClient.GetLayout(h.ctx)
+		assert.NoError(t, err)
+
+		expPeers := make([]garage.PeerLayout, len(allocs))
+		for i := range allocs {
+			expPeers[i] = garage.PeerLayout{
+				ID:       allocs[i].ID,
+				Capacity: 1_000_000_000,
+				Zone:     string(network.hostName),
+				Tags:     []string{},
+			}
+		}
+
+		assert.ElementsMatch(t, expPeers, layout.Peers)
+	})
+}

go/daemon/network/network_it_util_test.go

@@ -0,0 +1,368 @@
+package network
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"isle/bootstrap"
+	"isle/daemon/children"
+	"isle/daemon/daecommon"
+	"isle/nebula"
+	"isle/toolkit"
+	"os"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"testing"
+
+	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+// Utilities related to running network integration tests
+
+var (
+	getEnvBinDirPath = sync.OnceValue(func() string {
+		appDirPath := os.Getenv("APPDIR")
+		if appDirPath == "" {
+			panic("APPDIR not set")
+		}
+		return filepath.Join(appDirPath, "bin")
+	})
+
+	ipNetCounter          uint64 = 0
+	publicAddrPortCounter uint64 = 1024
+	tunDeviceCounter      uint64 = 0
+)
+
+func newIPNet() nebula.IPNet {
+	var (
+		ipNet    nebula.IPNet
+		ipNetStr = fmt.Sprintf(
+			"172.16.%d.0/24", atomic.AddUint64(&ipNetCounter, 1),
+		)
+	)
+
+	if err := ipNet.UnmarshalText([]byte(ipNetStr)); err != nil {
+		panic(fmt.Sprintf("parsing IPNet from %q: %v", ipNetStr, err))
+	}
+
+	return ipNet
+}
+
+func newPublicAddr() string {
+	return fmt.Sprintf(
+		"127.0.0.200:%d", atomic.AddUint64(&publicAddrPortCounter, 1),
+	)
+}
+
+func newTunDevice() string {
+	return fmt.Sprintf("isle-test-%d", atomic.AddUint64(&tunDeviceCounter, 1))
+}
+
+func mustParseNetworkConfigf(str string, args ...any) daecommon.NetworkConfig {
+	str = fmt.Sprintf(str, args...)
+
+	var networkConfig daecommon.NetworkConfig
+	if err := yaml.Unmarshal([]byte(str), &networkConfig); err != nil {
+		panic(fmt.Sprintf("parsing network config: %v", err))
+	}
+	return networkConfig
+}
+
+type integrationHarness struct {
+	ctx        context.Context
+	logger     *mlog.Logger
+	rootDir    toolkit.Dir
+	dirCounter uint64
+}
+
+func newIntegrationHarness(t *testing.T) *integrationHarness {
+	t.Parallel()
+	toolkit.MarkIntegrationTest(t)
+
+	rootDir, err := os.MkdirTemp("", "isle-network-it-test.*")
+	require.NoError(t, err)
+
+	t.Logf("Temporary test directory: %q", rootDir)
+
+	t.Cleanup(func() {
+		if t.Failed() {
+			t.Logf("Temp directory for failed test not deleted: %q", rootDir)
+			return
+		}
+
+		t.Logf("Deleting temp directory %q", rootDir)
+		assert.NoError(t, os.RemoveAll(rootDir))
+	})
+
+	return &integrationHarness{
+		ctx: context.Background(),
+		logger: mlog.NewLogger(&mlog.LoggerOpts{
+			MessageHandler: mlog.NewTestMessageHandler(t),
+		}),
+		rootDir: toolkit.Dir{Path: rootDir},
+	}
+}
+
+func (h *integrationHarness) mkDir(t *testing.T, name string) toolkit.Dir {
+	fullName := fmt.Sprintf("%s-%d", name, atomic.AddUint64(&h.dirCounter, 1))
+
+	t.Logf("Creating directory %q", fullName)
+	d, err := h.rootDir.MkChildDir(fullName, false)
+	require.NoError(t, err)
+
+	return d
+}
+
+type networkConfigOpts struct {
+	hasPublicAddr    bool
+	numStorageAllocs int
+}
+
+func (o *networkConfigOpts) withDefaults() *networkConfigOpts {
+	if o == nil {
+		o = new(networkConfigOpts)
+	}
+
+	return o
+}
+
+func (h *integrationHarness) mkNetworkConfig(
+	t *testing.T,
+	opts *networkConfigOpts,
+) daecommon.NetworkConfig {
+	if opts == nil {
+		opts = new(networkConfigOpts)
+	}
+
+	publicAddr := ""
+	if opts.hasPublicAddr {
+		publicAddr = newPublicAddr()
+	}
+
+	allocs := make([]map[string]any, opts.numStorageAllocs)
+	for i := range allocs {
+		allocs[i] = map[string]any{
+			"data_path": h.mkDir(t, "data").Path,
+			"meta_path": h.mkDir(t, "meta").Path,
+			"capacity":  1,
+		}
+	}
+
+	allocsJSON, err := json.Marshal(allocs)
+	require.NoError(t, err)
+
+	return mustParseNetworkConfigf(`
+        vpn:
+          public_addr: %q
+          tun:
+            device: %q
+        storage:
+          allocations: %s
+        `,
+		publicAddr,
+		newTunDevice(),
+		allocsJSON,
+	)
+}
+
+func (h *integrationHarness) mkChildrenOpts(
+	t *testing.T, runtimeDir toolkit.Dir,
+) *children.Opts {
+	var (
+		childrenLogFilePath = filepath.Join(runtimeDir.Path, "children.log")
+		childrenOpts        children.Opts
+	)
+
+	childrenLogFile, err := os.Create(childrenLogFilePath)
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		assert.NoError(t, err)
+	})
+
+	if os.Getenv("ISLE_INTEGRATION_TEST_CHILDREN_LOG_STDOUT") == "" {
+		childrenOpts = children.Opts{
+			Stdout: childrenLogFile,
+			Stderr: childrenLogFile,
+		}
+	} else {
+		childrenOpts = children.Opts{
+			Stdout: io.MultiWriter(os.Stdout, childrenLogFile),
+			Stderr: io.MultiWriter(os.Stdout, childrenLogFile),
+		}
+	}
+
+	return &childrenOpts
+}
+
+type createNetworkOpts struct {
+	creationParams bootstrap.CreationParams
+	manualShutdown bool
+}
+
+func (o *createNetworkOpts) withDefaults() *createNetworkOpts {
+	if o == nil {
+		o = new(createNetworkOpts)
+	}
+
+	if o.creationParams == (bootstrap.CreationParams{}) {
+		o.creationParams = bootstrap.NewCreationParams("test", "test.localnet")
+	}
+
+	return o
+}
+
+type integrationHarnessNetwork struct {
+	Network
+
+	hostName             nebula.HostName
+	creationParams       bootstrap.CreationParams
+	networkConfig        daecommon.NetworkConfig
+	stateDir, runtimeDir toolkit.Dir
+	opts                 *Opts
+}
+
+func (h *integrationHarness) createNetwork(
+	t *testing.T,
+	hostNameStr string,
+	opts *createNetworkOpts,
+) integrationHarnessNetwork {
+	t.Logf("Creating as %q", hostNameStr)
+	opts = opts.withDefaults()
+
+	var (
+		networkConfig = h.mkNetworkConfig(t, &networkConfigOpts{
+			hasPublicAddr:    true,
+			numStorageAllocs: 3,
+		})
+
+		stateDir   = h.mkDir(t, "state")
+		runtimeDir = h.mkDir(t, "runtime")
+
+		ipNet    = newIPNet()
+		hostName = nebula.HostName(hostNameStr)
+
+		networkOpts = &Opts{
+			ChildrenOpts:     h.mkChildrenOpts(t, runtimeDir),
+			GarageAdminToken: "admin_token",
+		}
+	)
+
+	network, err := Create(
+		h.ctx,
+		h.logger.WithNamespace("network").WithNamespace(hostNameStr),
+		networkConfig,
+		getEnvBinDirPath(),
+		stateDir,
+		runtimeDir,
+		opts.creationParams,
+		ipNet,
+		hostName,
+		networkOpts,
+	)
+	if err != nil {
+		t.Fatalf("creating Network: %v", err)
+	}
+
+	if !opts.manualShutdown {
+		t.Cleanup(func() {
+			t.Logf("Shutting down Network %q", hostNameStr)
+			if err := network.Shutdown(); err != nil {
+				t.Logf("Shutting down Network %q failed: %v", hostNameStr, err)
+			}
+		})
+	}
+
+	return integrationHarnessNetwork{
+		network,
+		hostName,
+		opts.creationParams,
+		networkConfig,
+		stateDir,
+		runtimeDir,
+		networkOpts,
+	}
+}
+
+type joinNetworkOpts struct {
+	*networkConfigOpts
+	canCreateHosts bool
+	manualShutdown bool
+}
+
+func (o *joinNetworkOpts) withDefaults() *joinNetworkOpts {
+	if o == nil {
+		o = new(joinNetworkOpts)
+	}
+
+	o.networkConfigOpts = o.networkConfigOpts.withDefaults()
+
+	return o
+}
+
+func (h *integrationHarness) joinNetwork(
+	t *testing.T,
+	network integrationHarnessNetwork,
+	hostNameStr string,
+	opts *joinNetworkOpts,
+) integrationHarnessNetwork {
+	opts = opts.withDefaults()
+	hostName := nebula.HostName(hostNameStr)
+
+	t.Logf("Creating bootstrap for %q", hostNameStr)
+	joiningBootstrap, err := network.CreateHost(h.ctx, hostName, CreateHostOpts{
+		CanCreateHosts: opts.canCreateHosts,
+	})
+	if err != nil {
+		t.Fatalf("creating host joining bootstrap: %v", err)
+	}
+
+	var (
+		networkConfig = h.mkNetworkConfig(t, opts.networkConfigOpts)
+		stateDir      = h.mkDir(t, "state")
+		runtimeDir    = h.mkDir(t, "runtime")
+		networkOpts   = &Opts{
+			ChildrenOpts:     h.mkChildrenOpts(t, runtimeDir),
+			GarageAdminToken: "admin_token",
+		}
+	)
+
+	t.Logf("Joining as %q", hostNameStr)
+	joinedNetwork, err := Join(
+		h.ctx,
+		h.logger.WithNamespace("network").WithNamespace(hostNameStr),
+		networkConfig,
+		joiningBootstrap,
+		getEnvBinDirPath(),
+		stateDir,
+		runtimeDir,
+		networkOpts,
+	)
+	if err != nil {
+		t.Fatalf("joining network: %v", err)
+	}
+
+	if !opts.manualShutdown {
+		t.Cleanup(func() {
+			t.Logf("Shutting down Network %q", hostNameStr)
+			if err := joinedNetwork.Shutdown(); err != nil {
+				t.Logf("Shutting down Network %q failed: %v", hostNameStr, err)
+			}
+		})
+	}
+
+	return integrationHarnessNetwork{
+		joinedNetwork,
+		hostName,
+		network.creationParams,
+		networkConfig,
+		stateDir,
+		runtimeDir,
+		networkOpts,
+	}
+}

go/dnsmasq/tpl.go

@@ -1,9 +1,14 @@
 package dnsmasq
 
 import (
-	"fmt"
-	"os"
+	"cmp"
+	"context"
+	"io"
+	"isle/toolkit"
+	"slices"
 	"text/template"
+
+	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
 )
 
 // ConfDataHost describes a host which can be resolved by dnsmasq.
@@ -44,22 +49,23 @@ server={{ . }}
 `))
 
 // WriteConfFile renders a dnsmasq.conf using the given data to a new
-// file at the given path.
-func WriteConfFile(path string, data ConfData) error {
-
-	file, err := os.OpenFile(
-		path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0640,
+// file at the given path, returning true if the file changed or didn't
+// previously exist.
+func WriteConfFile(
+	ctx context.Context, logger *mlog.Logger, path string, data ConfData,
+) (
+	bool, error,
+) {
+	slices.SortFunc(data.Hosts, func(i, j ConfDataHost) int {
+		return cmp.Or(
+			cmp.Compare(i.IP, j.IP),
+			cmp.Compare(i.Name, j.Name),
 		)
+	})
 
-	if err != nil {
-		return fmt.Errorf("creating file: %w", err)
-	}
-
-	defer file.Close()
-
-	if err := confTpl.Execute(file, data); err != nil {
-		return fmt.Errorf("rendering template to file: %w", err)
-	}
-
-	return nil
+	return toolkit.WriteFileCheckChanged(
+		ctx, logger, path, 0600, func(w io.Writer) error {
+			return confTpl.Execute(w, data)
+		},
+	)
 }

go/garage/admin_client.go

@@ -47,6 +47,8 @@ type AdminClient struct {
 	c          *http.Client
 	addr       string
 	adminToken string
+
+	transport *http.Transport
 }
 
 // NewAdminClient initializes and returns an AdminClient which will use the
@@ -54,16 +56,25 @@ type AdminClient struct {
 //
 // If Logger is nil then logs will be suppressed.
 func NewAdminClient(logger *mlog.Logger, addr, adminToken string) *AdminClient {
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+
 	return &AdminClient{
 		logger: logger,
 		c: &http.Client{
-			Transport: http.DefaultTransport.(*http.Transport).Clone(),
+			Transport: transport,
 		},
 		addr:       addr,
 		adminToken: adminToken,
+		transport:  transport,
 	}
 }
 
+// Close cleans up all resources held by the lient.
+func (c *AdminClient) Close() error {
+	c.transport.CloseIdleConnections()
+	return nil
+}
+
 // do performs an HTTP request with the given method (GET, POST) and path, and
 // using the json marshaling of the given body as the request body (unless body
 // is nil). It will JSON unmarshal the response into rcv, unless rcv is nil.
@@ -262,10 +273,23 @@ func (c *AdminClient) GrantBucketPermissions(
 // PeerLayout describes the properties of a garage peer in the context of the
 // layout of the cluster.
 type PeerLayout struct {
-	ID       string
-	Capacity int // Gb (SI units)
-	Zone     string
-	Tags     []string
+	ID       string   `json:"id"`
+	Capacity int      `json:"capacity"` // Gb (SI units)
+	Zone     string   `json:"zone"`
+	Tags     []string `json:"tags"`
+}
+
+// ClusterLayout describes the layout of the cluster as a whole.
+type ClusterLayout struct {
+	Peers []PeerLayout `json:"roles"`
+}
+
+// GetLayout returns the currently applied ClusterLayout.
+func (c *AdminClient) GetLayout(ctx context.Context) (ClusterLayout, error) {
+	// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Layout/operation/GetLayout
+	var res ClusterLayout
+	err := c.do(ctx, &res, "GET", "/v1/layout", nil)
+	return res, err
 }
 
 // ApplyLayout modifies the layout of the garage cluster. Only layout of the
@@ -273,21 +297,9 @@ type PeerLayout struct {
 func (c *AdminClient) ApplyLayout(
 	ctx context.Context, peers []PeerLayout,
 ) error {
-	type peerLayout struct {
-		ID       string   `json:"id"`
-		Capacity int      `json:"capacity"`
-		Zone     string   `json:"zone"`
-		Tags     []string `json:"tags"`
-	}
-
 	{
 		// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Layout/operation/ApplyLayout
-		clusterLayout := make([]peerLayout, len(peers))
-		for i := range peers {
-			clusterLayout[i] = peerLayout(peers[i])
-		}
-
-		err := c.do(ctx, nil, "POST", "/v1/layout", clusterLayout)
+		err := c.do(ctx, nil, "POST", "/v1/layout", peers)
 		if err != nil {
 			return fmt.Errorf("staging layout changes: %w", err)
 		}
@@ -296,7 +308,7 @@ func (c *AdminClient) ApplyLayout(
 	// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Layout/operation/GetLayout
 	var clusterLayout struct {
 		Version           int          `json:"version"`
-		StagedRoleChanges []peerLayout `json:"stagedRoleChanges"`
+		StagedRoleChanges []PeerLayout `json:"stagedRoleChanges"`
 	}
 
 	if err := c.do(ctx, &clusterLayout, "GET", "/v1/layout", nil); err != nil {

go/garage/client.go

@@ -3,6 +3,7 @@ package garage
 import (
 	"errors"
 	"fmt"
+	"net/http"
 
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
@@ -16,7 +17,16 @@ func IsKeyNotFound(err error) bool {
 }
 
 // S3APIClient is a client used to interact with garage's S3 API.
-type S3APIClient = *minio.Client
+type S3APIClient struct {
+	*minio.Client
+	transport *http.Transport
+}
+
+// Close cleans up all resources held by the client.
+func (c *S3APIClient) Close() error {
+	c.transport.CloseIdleConnections()
+	return nil
+}
 
 // S3APICredentials describe data fields necessary for authenticating with a
 // garage S3 API endpoint.
@@ -27,16 +37,21 @@ type S3APICredentials struct {
 
 // NewS3APIClient returns a minio client configured to use the given garage S3 API
 // endpoint.
-func NewS3APIClient(addr string, creds S3APICredentials) S3APIClient {
+func NewS3APIClient(addr string, creds S3APICredentials) *S3APIClient {
+	transport := http.DefaultTransport.(*http.Transport).Clone()
 
 	client, err := minio.New(addr, &minio.Options{
 		Creds:     credentials.NewStaticV4(creds.ID, creds.Secret, ""),
 		Region:    Region,
+		Transport: transport,
 	})
 
 	if err != nil {
 		panic(fmt.Sprintf("initializing minio client at addr %q and with creds %+v", addr, creds))
 	}
 
-	return client
+	return &S3APIClient{
+		Client:    client,
+		transport: transport,
+	}
 }

go/garage/garagesrv/tpl.go

@@ -1,13 +1,17 @@
 package garagesrv
 
 import (
-	"fmt"
+	"cmp"
+	"context"
 	"io"
-	"os"
+	"slices"
 	"strconv"
 	"text/template"
 
 	"isle/garage"
+	"isle/toolkit"
+
+	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
 )
 
 // GarageTomlData describes all fields needed for rendering a garage.toml
@@ -24,7 +28,6 @@ type GarageTomlData struct {
 }
 
 var garageTomlTpl = template.Must(template.New("").Parse(`
-
 metadata_dir = "{{ .MetaPath }}"
 data_dir = "{{ .DataPath }}"
 
@@ -45,7 +48,6 @@ s3_region = "garage"
 [admin]
 api_bind_addr = "{{ .AdminAddr }}"
 admin_token = "{{ .AdminToken }}"
-
 `))
 
 // RenderGarageToml renders a garage.toml using the given data into the writer.
@@ -54,22 +56,26 @@ func RenderGarageToml(into io.Writer, data GarageTomlData) error {
 }
 
 // WriteGarageTomlFile renders a garage.toml using the given data to a new file
-// at the given path.
-func WriteGarageTomlFile(path string, data GarageTomlData) error {
-
-	file, err := os.OpenFile(
-		path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600,
+// at the given path, returning true if the file changed or didn't
+// previously exist.
+func WriteGarageTomlFile(
+	ctx context.Context,
+	logger *mlog.Logger,
+	path string,
+	data GarageTomlData,
+) (
+	bool, error,
+) {
+	slices.SortFunc(data.BootstrapPeers, func(i, j garage.RemotePeer) int {
+		return cmp.Or(
+			cmp.Compare(i.IP, j.IP),
+			cmp.Compare(i.RPCPort, j.RPCPort),
 		)
+	})
 
-	if err != nil {
-		return fmt.Errorf("creating file: %w", err)
-	}
-
-	defer file.Close()
-
-	if err := garageTomlTpl.Execute(file, data); err != nil {
-		return fmt.Errorf("rendering template to file: %w", err)
-	}
-
-	return nil
+	return toolkit.WriteFileCheckChanged(
+		ctx, logger, path, 0600, func(w io.Writer) error {
+			return garageTomlTpl.Execute(w, data)
+		},
+	)
 }

go/go.mod

@@ -4,19 +4,21 @@ go 1.22
 
 require (
 	code.betamike.com/micropelago/pmux v0.0.0-20240719134913-f5fce902e8c4
-	dev.mediocregopher.com/mediocre-go-lib.git v0.0.0-20240511135822-4ab1176672d7
+	dev.mediocregopher.com/mediocre-go-lib.git v0.0.0-20241023182613-55984cdf5233
 	github.com/adrg/xdg v0.4.0
-	github.com/imdario/mergo v0.3.12
 	github.com/jxskiss/base62 v1.1.0
 	github.com/minio/minio-go/v7 v7.0.28
+	github.com/sergi/go-diff v1.3.1
 	github.com/slackhq/nebula v1.6.1
 	github.com/spf13/pflag v1.0.5
+	github.com/stretchr/testify v1.9.0
 	github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c
 	golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/google/uuid v1.1.1 // indirect
 	github.com/gopherjs/gopherjs v1.17.2 // indirect
@@ -29,6 +31,7 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rs/xid v1.2.1 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/smartystreets/assertions v1.13.0 // indirect

go/go.sum

@@ -1,7 +1,7 @@
 code.betamike.com/micropelago/pmux v0.0.0-20240719134913-f5fce902e8c4 h1:n4pGP12kgdH5kCTF4TZYpOjWuAR/zZLpM9j3neeVMEk=
 code.betamike.com/micropelago/pmux v0.0.0-20240719134913-f5fce902e8c4/go.mod h1:WlEWacLREVfIQl1IlBjKzuDgL56DFRvyl7YiL5gGP4w=
-dev.mediocregopher.com/mediocre-go-lib.git v0.0.0-20240511135822-4ab1176672d7 h1:wKQ3bXzG+KQDtRAN/xaRZ4aQtJe1pccleG6V43MvFxw=
-dev.mediocregopher.com/mediocre-go-lib.git v0.0.0-20240511135822-4ab1176672d7/go.mod h1:nP+AtQWrc3k5qq5y3ABiBLkOfUPlk/FO9fpTFpF+jgs=
+dev.mediocregopher.com/mediocre-go-lib.git v0.0.0-20241023182613-55984cdf5233 h1:Ea4HixNfDNDPh7zMngPpEeDf8gpociSPEROBFBedqIY=
+dev.mediocregopher.com/mediocre-go-lib.git v0.0.0-20241023182613-55984cdf5233/go.mod h1:nP+AtQWrc3k5qq5y3ABiBLkOfUPlk/FO9fpTFpF+jgs=
 github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls=
 github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -18,8 +18,6 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
 github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
-github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
-github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
@@ -31,6 +29,7 @@ github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e
 github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
 github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -53,6 +52,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
+github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/slackhq/nebula v1.6.1 h1:/OCTR3abj0Sbf2nGoLUrdDXImrCv0ZVFpVPP5qa0DsM=
@@ -66,9 +67,10 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
 github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
 golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 h1:tkVvjkPTB7pnW3jnid7kNyAMPVWllTNOf/qKDze4p9o=
@@ -88,12 +90,12 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
 gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

go/integration_test.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+cd "$SCRIPT_DIR" || exit
+
+this_user="$(whoami)"
+
+nix-shell -A testShell ../default.nix --run "
+    echo \"Requesting sudo in order to set thread capabilities, will drop back down to user '$this_user' immediately\"
+    sudo -E capsh \\
+        --caps=\"cap_net_admin,cap_net_bind_service+eip cap_setpcap,cap_setuid,cap_setgid+ep\" \\
+        --keep=1 \\
+        --user=\"$this_user\" \\
+        --addamb=cap_net_admin \\
+        --addamb=cap_net_bind_service \\
+        -- -c 'ISLE_INTEGRATION_TEST=1 go test $*'
+"

go/nebula/certificate.go

@@ -1,13 +1,35 @@
 package nebula
 
-import "github.com/slackhq/nebula/cert"
+import (
+	"fmt"
+
+	"github.com/slackhq/nebula/cert"
+)
 
 // Certificate wraps a NebulaCertificate to provide convenient (and consistent)
-// text (un)marshaling methods.
+// text (un)marshaling methods as well as normalization for equality checking.
 type Certificate struct {
 	inner cert.NebulaCertificate
 }
 
+// NewCertificate returns a Certificate wrapping the given one.
+func NewCertificate(inner cert.NebulaCertificate) (Certificate, error) {
+	// normalize the inner cert by marshaling to and unmarshaling from the PEM.
+	// This allows equality checking in tests to work between certs which have
+	// never been written to disk and those which have.
+	b, err := inner.MarshalToPEM()
+	if err != nil {
+		return Certificate{}, fmt.Errorf("marshaling to PEM: %w", err)
+	}
+
+	normInner, _, err := cert.UnmarshalNebulaCertificateFromPEM(b)
+	if err != nil {
+		return Certificate{}, fmt.Errorf("unmarshaling from PEM: %w", err)
+	}
+
+	return Certificate{inner: *normInner}, nil
+}
+
 // Unwrap returns the wrapped NebulaCertificate type.
 func (c Certificate) Unwrap() *cert.NebulaCertificate {
 	return &c.inner

go/nebula/nebula.go

@@ -90,7 +90,12 @@ func NewHostCert(
 		return Certificate{}, fmt.Errorf("signing host cert with ca.key: %w", err)
 	}
 
-	return Certificate{hostCert}, nil
+	c, err := NewCertificate(hostCert)
+	if err != nil {
+		return Certificate{}, fmt.Errorf("wrapping cert: %w", err)
+	}
+
+	return c, nil
 }
 
 // NewHostCredentials generates a new key/cert for a nebula host using the CA
@@ -136,7 +141,7 @@ func NewCACredentials(domain string, subnet IPNet) (CACredentials, error) {
 		expireAt                      = now.Add(2 * 365 * 24 * time.Hour)
 	)
 
-	caCert := cert.NebulaCertificate{
+	caCertInner := cert.NebulaCertificate{
 		Details: cert.NebulaCertificateDetails{
 			Name:      fmt.Sprintf("%s isle root cert", domain),
 			Subnets:   []*net.IPNet{(*net.IPNet)(&subnet)},
@@ -147,13 +152,18 @@ func NewCACredentials(domain string, subnet IPNet) (CACredentials, error) {
 		},
 	}
 
-	if err := signCert(&caCert, signingPrivKey); err != nil {
+	if err := signCert(&caCertInner, signingPrivKey); err != nil {
 		return CACredentials{}, fmt.Errorf("signing caCert: %w", err)
 	}
 
+	caCert, err := NewCertificate(caCertInner)
+	if err != nil {
+		return CACredentials{}, fmt.Errorf("wrapping caCert: %w", err)
+	}
+
 	return CACredentials{
 		Public: CAPublicCredentials{
-			Cert:       Certificate{caCert},
+			Cert:       caCert,
 			SigningKey: signingPubKey,
 		},
 		SigningPrivateKey: signingPrivKey,

go/toolkit/os.go

@@ -0,0 +1,84 @@
+package toolkit
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+
+	"dev.mediocregopher.com/mediocre-go-lib.git/mctx"
+	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
+	"github.com/sergi/go-diff/diffmatchpatch"
+)
+
+var differ = diffmatchpatch.New()
+
+// WriteFileCheckChanged uses the given callback to collect data to be written
+// to the given filepath with the given permission bits set. If the file's
+// contents have been modified (or if the file didn't previously exist) then
+// true is returned.
+func WriteFileCheckChanged(
+	ctx context.Context,
+	logger *mlog.Logger,
+	path string,
+	mode os.FileMode,
+	fn func(io.Writer) error,
+) (
+	bool, error,
+) {
+	var (
+		buf = new(bytes.Buffer)
+		h   = sha256.New()
+		w   = io.MultiWriter(buf, h)
+
+		changed bool
+	)
+
+	if err := fn(w); err != nil {
+		return false, fmt.Errorf("callback returned: %w", err)
+	}
+
+	f, err := os.Open(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		changed = true
+		// fine, we'll just write the file
+	} else if err != nil {
+		return false, fmt.Errorf("reading contents of existing file: %w", err)
+	} else {
+		var (
+			existingBuf = new(bytes.Buffer)
+			existingH   = sha256.New()
+			existingW   = io.MultiWriter(existingBuf, existingH)
+			_, copyErr  = io.Copy(existingW, f)
+			closeErr    = f.Close()
+		)
+
+		if err := errors.Join(closeErr, copyErr); err != nil {
+			return false, fmt.Errorf("hashing existing file: %w", err)
+		}
+
+		changed = !bytes.Equal(h.Sum(nil), existingH.Sum(nil))
+
+		if changed && logger.MaxLevel() >= mlog.LevelDebug.Int() {
+			var (
+				ctx  = mctx.Annotate(ctx, "path", path)
+				diff = differ.DiffPrettyText(
+					differ.DiffMain(existingBuf.String(), buf.String(), false),
+				)
+			)
+			logger.Debug(ctx, "WriteFileCheckChanged diff:\n"+diff)
+		}
+	}
+
+	if changed {
+		if err := os.WriteFile(path, buf.Bytes(), mode); err != nil {
+			return false, fmt.Errorf("writing file: %w", err)
+		}
+	}
+
+	return changed, nil
+}

go/toolkit/testutils.go

@@ -0,0 +1,14 @@
+package toolkit
+
+import (
+	"os"
+	"testing"
+)
+
+// MarkIntegrationTest marks a test as being an integration test. It will be
+// skipped if the ISLE_INTEGRATION_TEST envvar isn't set.
+func MarkIntegrationTest(t *testing.T) {
+	if os.Getenv("ISLE_INTEGRATION_TEST") == "" {
+		t.Skip("Skipped because ISLE_INTEGRATION_TEST isn't set")
+	}
+}

shell.nix

@@ -1,22 +1 @@
-{
-
-  buildSystem ? builtins.currentSystem,
-  pkgsNix ? (import ./nix/pkgs.nix),
-
-}: let
-
-  pkgs = pkgsNix.default {
-    inherit buildSystem;
-    hostSystem = buildSystem;
-  };
-
-in pkgs.mkShell {
-  buildInputs = [
-    pkgs.go
-    pkgs.golangci-lint
-    (pkgs.callPackage ./nix/gowrap.nix {})
-  ];
-  shellHook = ''
-    true # placeholder
-  '';
-}
+{}@args: ((import ./default.nix) args).devShell