Refactor children process reloading, add garage reloading

go/daemon/children/children.go

@@ -10,6 +10,7 @@ import (
 	"os"
 
 	"code.betamike.com/micropelago/pmux/pmuxlib"
+	"dev.mediocregopher.com/mediocre-go-lib.git/mctx"
 	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
 
 	"isle/bootstrap"
@@ -50,8 +51,11 @@ func (o *Opts) withDefaults() *Opts {
 type Children struct {
 	logger           *mlog.Logger
 	runtimeDir       toolkit.Dir
+	garageAdminToken string
 	opts             Opts
 
+	garageRPCSecret string
+
 	pmux *pmuxlib.Pmux
 }
 
@@ -81,7 +85,9 @@ func New(
 	c := &Children{
 		logger:           logger,
 		runtimeDir:       runtimeDir,
+		garageAdminToken: garageAdminToken,
 		opts:             *opts,
+		garageRPCSecret:  garageRPCSecret,
 	}
 
 	pmuxConfig, err := c.newPmuxConfig(
@@ -113,17 +119,22 @@ func New(
 // TODO block until process has been confirmed to have come back up
 // successfully.
 func (c *Children) reloadDNSMasq(
+	ctx context.Context,
 	networkConfig daecommon.NetworkConfig,
 	hostBootstrap bootstrap.Bootstrap,
 ) error {
-	_, err := dnsmasqWriteConfig(
+	if _, changed, err := dnsmasqWriteConfig(
-		c.runtimeDir.Path, networkConfig, hostBootstrap,
+		ctx, c.logger, c.runtimeDir.Path, networkConfig, hostBootstrap,
-	)
+	); err != nil {
-	if err != nil {
 		return fmt.Errorf("writing new dnsmasq config: %w", err)
+	} else if !changed {
+		c.logger.Info(ctx, "No changes to dnsmasq config file")
+		return nil
 	}
 
+	c.logger.Info(ctx, "dnsmasq config file has changed, restarting process")
 	c.pmux.Restart("dnsmasq")
+
 	return nil
 }
 
@@ -132,16 +143,18 @@ func (c *Children) reloadNebula(
 	networkConfig daecommon.NetworkConfig,
 	hostBootstrap bootstrap.Bootstrap,
 ) error {
-	_, err := nebulaWriteConfig(
+	if _, changed, err := nebulaWriteConfig(
-		c.runtimeDir.Path, networkConfig, hostBootstrap,
+		ctx, c.logger, c.runtimeDir.Path, networkConfig, hostBootstrap,
-	)
+	); err != nil {
-	if err != nil {
 		return fmt.Errorf("writing a new nebula config: %w", err)
+	} else if !changed {
+		c.logger.Info(ctx, "No changes to nebula config file")
+		return nil
 	}
 
+	c.logger.Info(ctx, "nebula config file has changed, restarting process")
 	c.pmux.Restart("nebula")
 
-	c.logger.Info(ctx, "Waiting for nebula VPN to come online")
 	if err := waitForNebula(ctx, c.logger, hostBootstrap); err != nil {
 		return fmt.Errorf("waiting for nebula to start: %w", err)
 	}
@@ -149,29 +162,80 @@ func (c *Children) reloadNebula(
 	return nil
 }
 
+// TODO this doesn't handle removing garage nodes
+func (c *Children) reloadGarage(
+	ctx context.Context,
+	networkConfig daecommon.NetworkConfig,
+	hostBootstrap bootstrap.Bootstrap,
+) error {
+	allocs := networkConfig.Storage.Allocations
+	if len(allocs) == 0 {
+		return nil
+	}
+
+	var anyChanged bool
+	for _, alloc := range allocs {
+		var (
+			procName = garagePmuxProcName(alloc)
+			ctx      = mctx.Annotate(
+				ctx,
+				"garageProcName", procName,
+				"garageDataPath", alloc.DataPath,
+			)
+		)
+
+		_, changed, err := garageWriteChildConfig(
+			ctx,
+			c.logger,
+			c.garageRPCSecret,
+			c.runtimeDir.Path,
+			c.garageAdminToken,
+			hostBootstrap,
+			alloc,
+		)
+		if err != nil {
+			return fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
+		} else if !changed {
+			c.logger.Info(ctx, "No changes to garage config file")
+			continue
+		}
+
+		anyChanged = true
+
+		c.logger.Info(ctx, "garage config has changed, restarting process")
+		c.pmux.Restart(procName)
+	}
+
+	if anyChanged {
+		if err := waitForGarage(
+			ctx, c.logger, networkConfig, c.garageAdminToken, hostBootstrap,
+		); err != nil {
+			return fmt.Errorf("waiting for garage to start: %w", err)
+		}
+	}
+
+	return nil
+}
+
 // Reload applies a ReloadDiff to the Children, using the given bootstrap which
 // must be the same one which was passed to CalculateReloadDiff.
 func (c *Children) Reload(
 	ctx context.Context,
 	newNetworkConfig daecommon.NetworkConfig,
 	newBootstrap bootstrap.Bootstrap,
-	diff ReloadDiff,
 ) error {
+	if err := c.reloadNebula(ctx, newNetworkConfig, newBootstrap); err != nil {
+		return fmt.Errorf("reloading nebula: %w", err)
+	}
+
 	var errs []error
 
-	if diff.DNSChanged {
+	if err := c.reloadDNSMasq(ctx, newNetworkConfig, newBootstrap); err != nil {
-		c.logger.Info(ctx, "Reloading dnsmasq to account for bootstrap changes")
-		if err := c.reloadDNSMasq(newNetworkConfig, newBootstrap); err != nil {
 		errs = append(errs, fmt.Errorf("reloading dnsmasq: %w", err))
 	}
-	}
 
-	if diff.NebulaChanged {
+	if err := c.reloadGarage(ctx, newNetworkConfig, newBootstrap); err != nil {
-		c.logger.Info(ctx, "Reloading nebula to account for bootstrap changes")
+		errs = append(errs, fmt.Errorf("reloading garage: %w", err))
-		err := c.reloadNebula(ctx, newNetworkConfig, newBootstrap)
-		if err != nil {
-			errs = append(errs, fmt.Errorf("reloading nebula: %w", err))
-		}
 	}
 
 	return errors.Join(errs...)

go/daemon/children/diff.go

@@ -1,47 +0,0 @@
-package children
-
-import (
-	"errors"
-	"fmt"
-	"isle/bootstrap"
-	"isle/daemon/daecommon"
-	"reflect"
-)
-
-// ReloadDiff describes which children had their configurations changed as part
-// of a change in the bootstrap.
-type ReloadDiff struct {
-	NebulaChanged bool
-	DNSChanged    bool
-}
-
-// CalculateReloadDiff calculates a ReloadDiff based on an old and new
-// bootstrap.
-func CalculateReloadDiff(
-	networkConfig daecommon.NetworkConfig,
-	prevBootstrap, nextBootstrap bootstrap.Bootstrap,
-) (
-	diff ReloadDiff, err error,
-) {
-	{
-		prevNebulaConfig, prevErr := nebulaConfig(networkConfig, prevBootstrap)
-		nextNebulaConfig, nextErr := nebulaConfig(networkConfig, nextBootstrap)
-		if err = errors.Join(prevErr, nextErr); err != nil {
-			err = fmt.Errorf("calculating nebula config: %w", err)
-			return
-		}
-
-		diff.NebulaChanged = !reflect.DeepEqual(
-			prevNebulaConfig, nextNebulaConfig,
-		)
-	}
-
-	{
-		diff.DNSChanged = !reflect.DeepEqual(
-			dnsmasqConfig(networkConfig, prevBootstrap),
-			dnsmasqConfig(networkConfig, nextBootstrap),
-		)
-	}
-
-	return
-}

go/daemon/children/dnsmasq.go

@@ -7,55 +7,50 @@ import (
 	"isle/daemon/daecommon"
 	"isle/dnsmasq"
 	"path/filepath"
-	"sort"
 
 	"code.betamike.com/micropelago/pmux/pmuxlib"
 	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
 )
 
-func dnsmasqConfig(
+func dnsmasqWriteConfig(
-	networkConfig daecommon.NetworkConfig, hostBootstrap bootstrap.Bootstrap,
+	ctx context.Context,
-) dnsmasq.ConfData {
+	logger *mlog.Logger,
-	hostsSlice := make([]dnsmasq.ConfDataHost, 0, len(hostBootstrap.Hosts))
+	runtimeDirPath string,
+	networkConfig daecommon.NetworkConfig,
+	hostBootstrap bootstrap.Bootstrap,
+) (
+	string, bool, error,
+) {
+	hosts := make([]dnsmasq.ConfDataHost, 0, len(hostBootstrap.Hosts))
 	for _, host := range hostBootstrap.Hosts {
-		hostsSlice = append(hostsSlice, dnsmasq.ConfDataHost{
+		hosts = append(hosts, dnsmasq.ConfDataHost{
 			Name: string(host.Name),
 			IP:   host.IP().String(),
 		})
 	}
 
-	sort.Slice(hostsSlice, func(i, j int) bool {
+	var (
-		return hostsSlice[i].IP < hostsSlice[j].IP
+		confPath = filepath.Join(runtimeDirPath, "dnsmasq.conf")
-	})
+		confData = dnsmasq.ConfData{
-
-	return dnsmasq.ConfData{
 			Resolvers: networkConfig.DNS.Resolvers,
 			Domain:    hostBootstrap.NetworkCreationParams.Domain,
 			IP:        hostBootstrap.ThisHost().IP().String(),
-		Hosts:     hostsSlice,
+			Hosts:     hosts,
 		}
-}
-
-func dnsmasqWriteConfig(
-	runtimeDirPath string,
-	networkConfig daecommon.NetworkConfig,
-	hostBootstrap bootstrap.Bootstrap,
-) (
-	string, error,
-) {
-	var (
-		confPath = filepath.Join(runtimeDirPath, "dnsmasq.conf")
-		confData = dnsmasqConfig(networkConfig, hostBootstrap)
 	)
 
-	if err := dnsmasq.WriteConfFile(confPath, confData); err != nil {
+	changed, err := dnsmasq.WriteConfFile(ctx, logger, confPath, confData)
-		return "", fmt.Errorf("writing dnsmasq.conf to %q: %w", confPath, err)
+	if err != nil {
+		return "", false, fmt.Errorf(
+			"writing dnsmasq.conf to %q: %w", confPath, err,
+		)
 	}
 
-	return confPath, nil
+	return confPath, changed, nil
 }
 
 func dnsmasqPmuxProcConfig(
+	ctx context.Context,
 	logger *mlog.Logger,
 	runtimeDirPath, binDirPath string,
 	networkConfig daecommon.NetworkConfig,
@@ -63,8 +58,8 @@ func dnsmasqPmuxProcConfig(
 ) (
 	pmuxlib.ProcessConfig, error,
 ) {
-	confPath, err := dnsmasqWriteConfig(
+	confPath, _, err := dnsmasqWriteConfig(
-		runtimeDirPath, networkConfig, hostBootstrap,
+		ctx, logger, runtimeDirPath, networkConfig, hostBootstrap,
 	)
 	if err != nil {
 		return pmuxlib.ProcessConfig{}, fmt.Errorf(

go/daemon/children/garage.go

@@ -2,6 +2,7 @@ package children
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"isle/bootstrap"
 	"isle/daemon/daecommon"
@@ -29,9 +30,6 @@ func waitForGarage(
 ) error {
 
 	allocs := networkConfig.Storage.Allocations
-
-	// if this host doesn't have any allocations specified then fall back to
-	// waiting for nebula
 	if len(allocs) == 0 {
 		return nil
 	}
@@ -49,8 +47,10 @@ func waitForGarage(
 			adminClientLogger, adminAddr, adminToken,
 		)
 
-		ctx := mctx.Annotate(ctx, "garageAdminAddr", adminAddr)
+		ctx := mctx.Annotate(
-		logger.Debug(ctx, "waiting for garage instance to start")
+			ctx, "garageAdminAddr", adminAddr, "garageDataPath", alloc.DataPath,
+		)
+		logger.Info(ctx, "Waiting for garage instance to be healthy")
 
 		if err := adminClient.Wait(ctx); err != nil {
 			return fmt.Errorf("waiting for garage instance %q to start up: %w", adminAddr, err)
@@ -64,17 +64,19 @@ func waitForGarage(
 }
 
 func garageWriteChildConfig(
+	ctx context.Context,
+	logger *mlog.Logger,
 	rpcSecret, runtimeDirPath, adminToken string,
 	hostBootstrap bootstrap.Bootstrap,
 	alloc daecommon.ConfigStorageAllocation,
 ) (
-	string, error,
+	string, bool, error,
 ) {
+	var (
+		thisHost = hostBootstrap.ThisHost()
+		id       = daecommon.BootstrapGarageHostForAlloc(thisHost, alloc).ID
 
-	thisHost := hostBootstrap.ThisHost()
+		peer = garage.LocalPeer{
-	id := daecommon.BootstrapGarageHostForAlloc(thisHost, alloc).ID
-
-	peer := garage.LocalPeer{
 			RemotePeer: garage.RemotePeer{
 				ID:        id,
 				IP:        thisHost.IP().String(),
@@ -84,11 +86,16 @@ func garageWriteChildConfig(
 			AdminPort: alloc.AdminPort,
 		}
 
-	garageTomlPath := filepath.Join(
+		garageTomlPath = filepath.Join(
 			runtimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
 		)
+	)
 
-	err := garagesrv.WriteGarageTomlFile(garageTomlPath, garagesrv.GarageTomlData{
+	changed, err := garagesrv.WriteGarageTomlFile(
+		ctx,
+		logger,
+		garageTomlPath,
+		garagesrv.GarageTomlData{
 			MetaPath: alloc.MetaPath,
 			DataPath: alloc.DataPath,
 
@@ -97,13 +104,20 @@ func garageWriteChildConfig(
 
 			LocalPeer:      peer,
 			BootstrapPeers: hostBootstrap.GaragePeers(),
-	})
+		},
+	)
 
 	if err != nil {
-		return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
+		return "", false, fmt.Errorf(
+			"creating garage.toml file at %q: %w", garageTomlPath, err,
+		)
 	}
 
-	return garageTomlPath, nil
+	return garageTomlPath, changed, nil
+}
+
+func garagePmuxProcName(alloc daecommon.ConfigStorageAllocation) string {
+	return fmt.Sprintf("garage-%d", alloc.RPCPort)
 }
 
 func garagePmuxProcConfigs(
@@ -122,20 +136,22 @@ func garagePmuxProcConfigs(
 	)
 
 	if len(allocs) > 0 && rpcSecret == "" {
-		logger.WarnString(ctx, "Not starting garage instances for storage allocations, missing garage RPC secret")
+		return nil, errors.New("Storage allocations defined, but garage RPC secret is not available")
-		return nil, nil
 	}
 
 	for _, alloc := range allocs {
-
+		childConfigPath, _, err := garageWriteChildConfig(
-		childConfigPath, err := garageWriteChildConfig(
+			ctx,
-			rpcSecret, runtimeDirPath, adminToken, hostBootstrap, alloc,
+			logger,
+			rpcSecret, runtimeDirPath, adminToken,
+			hostBootstrap,
+			alloc,
 		)
 		if err != nil {
 			return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
 		}
 
-		procName := fmt.Sprintf("garage-%d", alloc.RPCPort)
+		procName := garagePmuxProcName(alloc)
 		pmuxProcConfigs[procName] = pmuxlib.ProcessConfig{
 			Cmd:  filepath.Join(binDirPath, "garage"),
 			Args: []string{"-c", childConfigPath, "server"},

go/daemon/children/nebula.go

@@ -3,9 +3,10 @@ package children
 import (
 	"context"
 	"fmt"
+	"io"
 	"isle/bootstrap"
 	"isle/daemon/daecommon"
-	"isle/yamlutil"
+	"isle/toolkit"
 	"net"
 	"path/filepath"
 
@@ -13,6 +14,7 @@ import (
 	"dev.mediocregopher.com/mediocre-go-lib.git/mctx"
 	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
 	"github.com/slackhq/nebula/cert"
+	"gopkg.in/yaml.v3"
 )
 
 // waitForNebula waits for the nebula interface to have been started up. It does
@@ -33,7 +35,7 @@ func waitForNebula(
 	until(
 		ctx,
 		logger,
-		"Creating UDP socket from nebula addr",
+		"Checking if nebula is online by creating UDP socket from nebula IP",
 		func(context.Context) error {
 			conn, err := net.DialUDP("udp", lUDPAddr, rUDPAddr)
 			if err != nil {
@@ -47,6 +49,7 @@ func waitForNebula(
 	return ctx.Err()
 }
 
+// TODO this needs to be produce a deterministic config value.
 func nebulaConfig(
 	networkConfig daecommon.NetworkConfig,
 	hostBootstrap bootstrap.Bootstrap,
@@ -142,44 +145,50 @@ func nebulaConfig(
 }
 
 func nebulaWriteConfig(
+	ctx context.Context,
+	logger *mlog.Logger,
 	runtimeDirPath string,
 	networkConfig daecommon.NetworkConfig,
 	hostBootstrap bootstrap.Bootstrap,
 ) (
-	string, error,
+	string, bool, error,
 ) {
 	config, err := nebulaConfig(networkConfig, hostBootstrap)
 	if err != nil {
-		return "", fmt.Errorf("creating nebula config: %w", err)
+		return "", false, fmt.Errorf("creating nebula config: %w", err)
 	}
 
 	nebulaYmlPath := filepath.Join(runtimeDirPath, "nebula.yml")
 
-	if err := yamlutil.WriteYamlFile(config, nebulaYmlPath, 0600); err != nil {
+	changed, err := toolkit.WriteFileCheckChanged(
-		return "", fmt.Errorf("writing nebula.yml to %q: %w", nebulaYmlPath, err)
+		ctx, logger, nebulaYmlPath, 0600, func(w io.Writer) error {
+			return yaml.NewEncoder(w).Encode(config)
+		},
+	)
+	if err != nil {
+		return "", false, fmt.Errorf(
+			"writing nebula.yml to %q: %w", nebulaYmlPath, err,
+		)
 	}
 
-	return nebulaYmlPath, nil
+	return nebulaYmlPath, changed, nil
 }
 
 func nebulaPmuxProcConfig(
+	ctx context.Context,
+	logger *mlog.Logger,
 	runtimeDirPath, binDirPath string,
 	networkConfig daecommon.NetworkConfig,
 	hostBootstrap bootstrap.Bootstrap,
 ) (
 	pmuxlib.ProcessConfig, error,
 ) {
-	config, err := nebulaConfig(networkConfig, hostBootstrap)
+	nebulaYmlPath, _, err := nebulaWriteConfig(
+		ctx, logger, runtimeDirPath, networkConfig, hostBootstrap,
+	)
 	if err != nil {
 		return pmuxlib.ProcessConfig{}, fmt.Errorf(
-			"creating nebula config: %w", err,
+			"writing nebula config: %w", err,
-		)
-	}
-
-	nebulaYmlPath := filepath.Join(runtimeDirPath, "nebula.yml")
-	if err := yamlutil.WriteYamlFile(config, nebulaYmlPath, 0600); err != nil {
-		return pmuxlib.ProcessConfig{}, fmt.Errorf(
-			"writing nebula.yml to %q: %w", nebulaYmlPath, err,
 		)
 	}
 

go/daemon/children/pmux.go

@@ -19,6 +19,8 @@ func (c *Children) newPmuxConfig(
 	pmuxlib.Config, error,
 ) {
 	nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(
+		ctx,
+		c.logger,
 		c.runtimeDir.Path,
 		binDirPath,
 		networkConfig,
@@ -29,6 +31,7 @@ func (c *Children) newPmuxConfig(
 	}
 
 	dnsmasqPmuxProcConfig, err := dnsmasqPmuxProcConfig(
+		ctx,
 		c.logger,
 		c.runtimeDir.Path,
 		binDirPath,
@@ -72,12 +75,10 @@ func (c *Children) postPmuxInit(
 	garageAdminToken string,
 	hostBootstrap bootstrap.Bootstrap,
 ) error {
-	c.logger.Info(ctx, "Waiting for nebula VPN to come online")
 	if err := waitForNebula(ctx, c.logger, hostBootstrap); err != nil {
 		return fmt.Errorf("waiting for nebula to start: %w", err)
 	}
 
-	c.logger.Info(ctx, "Waiting for garage instances to come online")
 	err := waitForGarage(
 		ctx, c.logger, networkConfig, garageAdminToken, hostBootstrap,
 	)

go/daemon/network/network.go

@@ -460,8 +460,8 @@ func (n *network) initialize(
 		return fmt.Errorf("Reloading network bootstrap: %w", err)
 	}
 
-	// TODO annotate this context with creation params
+	ctx = context.WithoutCancel(ctx)
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(ctx)
 	n.wg.Add(1)
 	go func() {
 		defer n.wg.Done()
@@ -473,7 +473,7 @@ func (n *network) initialize(
 	go func() {
 		defer n.wg.Done()
 		n.reloadLoop(ctx)
-		n.logger.Debug(ctx, "Daemon restart loop stopped")
+		n.logger.Debug(ctx, "Daemon reload loop stopped")
 	}()
 
 	return nil
@@ -603,23 +603,13 @@ func (n *network) reload(
 		return fmt.Errorf("writing bootstrap to state dir: %w", err)
 	}
 
-	diff, err := children.CalculateReloadDiff(
-		n.networkConfig, n.currBootstrap, newBootstrap,
-	)
-	if err != nil {
-		return fmt.Errorf("calculating diff between bootstraps: %w", err)
-	} else if diff == (children.ReloadDiff{}) {
-		n.logger.Info(ctx, "No changes to bootstrap detected")
-		return nil
-	}
-
 	n.networkConfig = newNetworkConfig
 	n.currBootstrap = newBootstrap
 
 	n.logger.Info(ctx, "Reloading child processes")
-	err = n.children.Reload(ctx, newNetworkConfig, newBootstrap, diff)
+	err = n.children.Reload(ctx, newNetworkConfig, newBootstrap)
 	if err != nil {
-		return fmt.Errorf("reloading child processes (diff:%+v): %w", diff, err)
+		return fmt.Errorf("reloading child processes: %w", err)
 	}
 
 	return nil

go/daemon/network/network_it_util_test.go

@@ -123,6 +123,14 @@ type networkConfigOpts struct {
 	numStorageAllocs int
 }
 
+func (o *networkConfigOpts) withDefaults() *networkConfigOpts {
+	if o == nil {
+		o = new(networkConfigOpts)
+	}
+
+	return o
+}
+
 func (h *integrationHarness) mkNetworkConfig(
 	t *testing.T,
 	opts *networkConfigOpts,
@@ -282,31 +290,31 @@ func (h *integrationHarness) createNetwork(
 }
 
 type joinNetworkOpts struct {
-	networkConfigOpts
+	*networkConfigOpts
 	canCreateHosts bool
 	manualShutdown bool
 }
 
+func (o *joinNetworkOpts) withDefaults() *joinNetworkOpts {
+	if o == nil {
+		o = new(joinNetworkOpts)
+	}
+
+	o.networkConfigOpts = o.networkConfigOpts.withDefaults()
+
+	return o
+}
+
 func (h *integrationHarness) joinNetwork(
 	t *testing.T,
 	network integrationHarnessNetwork,
 	hostNameStr string,
 	opts *joinNetworkOpts,
 ) integrationHarnessNetwork {
-	t.Logf("Joining as %q", hostNameStr)
+	opts = opts.withDefaults()
-	opts = new(joinNetworkOpts)
+	hostName := nebula.HostName(hostNameStr)
-
-	var (
-		networkConfig = h.mkNetworkConfig(t, &opts.networkConfigOpts)
-		stateDir      = h.mkDir(t, "state")
-		runtimeDir    = h.mkDir(t, "runtime")
-		hostName      = nebula.HostName(hostNameStr)
-		networkOpts   = &Opts{
-			ChildrenOpts:     h.mkChildrenOpts(t, runtimeDir),
-			GarageAdminToken: "admin_token",
-		}
-	)
 
+	t.Logf("Creating bootstrap for %q", hostNameStr)
 	joiningBootstrap, err := network.CreateHost(h.ctx, hostName, CreateHostOpts{
 		CanCreateHosts: opts.canCreateHosts,
 	})
@@ -314,6 +322,17 @@ func (h *integrationHarness) joinNetwork(
 		t.Fatalf("creating host joining bootstrap: %v", err)
 	}
 
+	var (
+		networkConfig = h.mkNetworkConfig(t, opts.networkConfigOpts)
+		stateDir      = h.mkDir(t, "state")
+		runtimeDir    = h.mkDir(t, "runtime")
+		networkOpts   = &Opts{
+			ChildrenOpts:     h.mkChildrenOpts(t, runtimeDir),
+			GarageAdminToken: "admin_token",
+		}
+	)
+
+	t.Logf("Joining as %q", hostNameStr)
 	joinedNetwork, err := Join(
 		h.ctx,
 		h.logger.WithNamespace("network").WithNamespace(hostNameStr),

go/dnsmasq/tpl.go

@@ -1,9 +1,14 @@
 package dnsmasq
 
 import (
-	"fmt"
+	"cmp"
-	"os"
+	"context"
+	"io"
+	"isle/toolkit"
+	"slices"
 	"text/template"
+
+	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
 )
 
 // ConfDataHost describes a host which can be resolved by dnsmasq.
@@ -44,22 +49,23 @@ server={{ . }}
 `))
 
 // WriteConfFile renders a dnsmasq.conf using the given data to a new
-// file at the given path.
+// file at the given path, returning true if the file changed or didn't
-func WriteConfFile(path string, data ConfData) error {
+// previously exist.
-
+func WriteConfFile(
-	file, err := os.OpenFile(
+	ctx context.Context, logger *mlog.Logger, path string, data ConfData,
-		path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0640,
+) (
+	bool, error,
+) {
+	slices.SortFunc(data.Hosts, func(i, j ConfDataHost) int {
+		return cmp.Or(
+			cmp.Compare(i.IP, j.IP),
+			cmp.Compare(i.Name, j.Name),
 		)
+	})
 
-	if err != nil {
+	return toolkit.WriteFileCheckChanged(
-		return fmt.Errorf("creating file: %w", err)
+		ctx, logger, path, 0600, func(w io.Writer) error {
-	}
+			return confTpl.Execute(w, data)
-
+		},
-	defer file.Close()
+	)
-
-	if err := confTpl.Execute(file, data); err != nil {
-		return fmt.Errorf("rendering template to file: %w", err)
-	}
-
-	return nil
 }

go/garage/garagesrv/tpl.go

@@ -1,13 +1,17 @@
 package garagesrv
 
 import (
-	"fmt"
+	"cmp"
+	"context"
 	"io"
-	"os"
+	"slices"
 	"strconv"
 	"text/template"
 
 	"isle/garage"
+	"isle/toolkit"
+
+	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
 )
 
 // GarageTomlData describes all fields needed for rendering a garage.toml
@@ -24,7 +28,6 @@ type GarageTomlData struct {
 }
 
 var garageTomlTpl = template.Must(template.New("").Parse(`
-
 metadata_dir = "{{ .MetaPath }}"
 data_dir = "{{ .DataPath }}"
 
@@ -45,7 +48,6 @@ s3_region = "garage"
 [admin]
 api_bind_addr = "{{ .AdminAddr }}"
 admin_token = "{{ .AdminToken }}"
-
 `))
 
 // RenderGarageToml renders a garage.toml using the given data into the writer.
@@ -54,22 +56,26 @@ func RenderGarageToml(into io.Writer, data GarageTomlData) error {
 }
 
 // WriteGarageTomlFile renders a garage.toml using the given data to a new file
-// at the given path.
+// at the given path, returning true if the file changed or didn't
-func WriteGarageTomlFile(path string, data GarageTomlData) error {
+// previously exist.
-
+func WriteGarageTomlFile(
-	file, err := os.OpenFile(
+	ctx context.Context,
-		path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600,
+	logger *mlog.Logger,
+	path string,
+	data GarageTomlData,
+) (
+	bool, error,
+) {
+	slices.SortFunc(data.BootstrapPeers, func(i, j garage.RemotePeer) int {
+		return cmp.Or(
+			cmp.Compare(i.IP, j.IP),
+			cmp.Compare(i.RPCPort, j.RPCPort),
 		)
+	})
 
-	if err != nil {
+	return toolkit.WriteFileCheckChanged(
-		return fmt.Errorf("creating file: %w", err)
+		ctx, logger, path, 0600, func(w io.Writer) error {
-	}
+			return garageTomlTpl.Execute(w, data)
-
+		},
-	defer file.Close()
+	)
-
-	if err := garageTomlTpl.Execute(file, data); err != nil {
-		return fmt.Errorf("rendering template to file: %w", err)
-	}
-
-	return nil
 }

go/go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/adrg/xdg v0.4.0
 	github.com/jxskiss/base62 v1.1.0
 	github.com/minio/minio-go/v7 v7.0.28
+	github.com/sergi/go-diff v1.3.1
 	github.com/slackhq/nebula v1.6.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0

go/go.sum

@@ -29,6 +29,7 @@ github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e
 github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
 github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -51,6 +52,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
+github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/slackhq/nebula v1.6.1 h1:/OCTR3abj0Sbf2nGoLUrdDXImrCv0ZVFpVPP5qa0DsM=
@@ -64,6 +67,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
@@ -86,10 +90,13 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
 gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

go/toolkit/os.go

@@ -0,0 +1,84 @@
+package toolkit
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+
+	"dev.mediocregopher.com/mediocre-go-lib.git/mctx"
+	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
+	"github.com/sergi/go-diff/diffmatchpatch"
+)
+
+var differ = diffmatchpatch.New()
+
+// WriteFileCheckChanged uses the given callback to collect data to be written
+// to the given filepath with the given permission bits set. If the file's
+// contents have been modified (or if the file didn't previously exist) then
+// true is returned.
+func WriteFileCheckChanged(
+	ctx context.Context,
+	logger *mlog.Logger,
+	path string,
+	mode os.FileMode,
+	fn func(io.Writer) error,
+) (
+	bool, error,
+) {
+	var (
+		buf = new(bytes.Buffer)
+		h   = sha256.New()
+		w   = io.MultiWriter(buf, h)
+
+		changed bool
+	)
+
+	if err := fn(w); err != nil {
+		return false, fmt.Errorf("callback returned: %w", err)
+	}
+
+	f, err := os.Open(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		changed = true
+		// fine, we'll just write the file
+	} else if err != nil {
+		return false, fmt.Errorf("reading contents of existing file: %w", err)
+	} else {
+		var (
+			existingBuf = new(bytes.Buffer)
+			existingH   = sha256.New()
+			existingW   = io.MultiWriter(existingBuf, existingH)
+			_, copyErr  = io.Copy(existingW, f)
+			closeErr    = f.Close()
+		)
+
+		if err := errors.Join(closeErr, copyErr); err != nil {
+			return false, fmt.Errorf("hashing existing file: %w", err)
+		}
+
+		changed = !bytes.Equal(h.Sum(nil), existingH.Sum(nil))
+
+		if changed && logger.MaxLevel() >= mlog.LevelDebug.Int() {
+			var (
+				ctx  = mctx.Annotate(ctx, "path", path)
+				diff = differ.DiffPrettyText(
+					differ.DiffMain(existingBuf.String(), buf.String(), false),
+				)
+			)
+			logger.Debug(ctx, "WriteFileCheckChanged diff:\n"+diff)
+		}
+	}
+
+	if changed {
+		if err := os.WriteFile(path, buf.Bytes(), mode); err != nil {
+			return false, fmt.Errorf("writing file: %w", err)
+		}
+	}
+
+	return changed, nil
+}