go/bootstrap/bootstrap.go

@@ -93,18 +93,12 @@ func New(
 	}, nil
 }
 
-// FromFile reads a bootstrap from a file at the given path. The HostAssigned
+// FromReader reads a bootstrap file from the given io.Reader.
-// field will automatically be unwrapped.
+func FromReader(r io.Reader) (Bootstrap, error) {
-func FromFile(path string) (Bootstrap, error) {
-	f, err := os.Open(path)
-	if err != nil {
-		return Bootstrap{}, fmt.Errorf("opening file: %w", err)
-	}
-	defer f.Close()
-
 	var b Bootstrap
 
-	if err := json.NewDecoder(f).Decode(&b); err != nil {
+	err := json.NewDecoder(r).Decode(&b)
+	if err != nil {
 		return Bootstrap{}, fmt.Errorf("decoding json: %w", err)
 	}
 
@@ -112,7 +106,19 @@ func FromFile(path string) (Bootstrap, error) {
 		return Bootstrap{}, fmt.Errorf("unwrapping host assigned: %w", err)
 	}
 
-	return b, nil
+	return b, err
+}
+
+// FromFile reads a bootstrap from a file at the given path.
+func FromFile(path string) (Bootstrap, error) {
+
+	f, err := os.Open(path)
+	if err != nil {
+		return Bootstrap{}, fmt.Errorf("opening file: %w", err)
+	}
+	defer f.Close()
+
+	return FromReader(f)
 }
 
 // WriteTo writes the Bootstrap as a new bootstrap to the given io.Writer.

go/cmd/entrypoint/admin.go

@@ -171,7 +171,9 @@ var subCmdAdminCreateNetwork = subCmd{
 			logger.WithNamespace("daemon"),
 			daemonConfig,
 			hostBootstrap,
+			envRuntimeDirPath,
 			envBinDirPath,
+			envStateDirPath,
 			&daemon.Opts{
 				// SkipHostBootstrapPush is required, because the global bucket
 				// hasn't actually been initialized yet, so there's nowhere to
@@ -199,7 +201,7 @@ var subCmdAdminCreateNetwork = subCmd{
 			return fmt.Errorf("initializing garage shared global bucket: %w", err)
 		}
 
-		if err := daemonInst.Shutdown(); err != nil {
+		if err := daemonInst.Shutdown(ctx); err != nil {
 			return fmt.Errorf("shutting down daemon: %w (this can mean there are zombie children leftover)", err)
 		}
 

go/cmd/entrypoint/bootstrap_util.go

@@ -11,7 +11,7 @@ import (
 
 func loadHostBootstrap() (bootstrap.Bootstrap, error) {
 
-	stateDirPath := bootstrap.StateDirPath(daemonEnvVars.StateDirPath)
+	stateDirPath := bootstrap.StateDirPath(envStateDirPath)
 
 	hostBootstrap, err := bootstrap.FromFile(stateDirPath)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -29,7 +29,7 @@ func loadHostBootstrap() (bootstrap.Bootstrap, error) {
 
 func writeBootstrapToStateDir(hostBootstrap bootstrap.Bootstrap) error {
 
-	path := bootstrap.StateDirPath(daemonEnvVars.StateDirPath)
+	path := bootstrap.StateDirPath(envStateDirPath)
 	dirPath := filepath.Dir(path)
 
 	if err := os.MkdirAll(dirPath, 0700); err != nil {

go/cmd/entrypoint/daemon.go

@@ -1,11 +1,13 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
 	"io/fs"
 	"os"
+	"time"
 
 	"isle/bootstrap"
 	"isle/daemon"
@@ -14,6 +16,153 @@ import (
 	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
 )
 
+// The daemon sub-command deals with starting an actual isle daemon
+// process, which is required to be running for most other Isle
+// functionality. The sub-command does the following:
+//
+// * Creates and locks the runtime directory.
+//
+// * Creates the data directory and copies the appdir bootstrap file into there,
+// if it's not already there.
+//
+// * Merges daemon configuration into the bootstrap configuration, and rewrites
+// the bootstrap file.
+//
+// * Sets up environment variables that all other sub-processes then use, based
+// on the runtime dir.
+//
+// * Dynamically creates the root pmux config and runs pmux.
+//
+// * (On exit) cleans up the runtime directory.
+
+// creates a new bootstrap file using available information from the network. If
+// the new bootstrap file is different than the existing one, the existing one
+// is overwritten and true is returned.
+func reloadBootstrap(
+	ctx context.Context,
+	logger *mlog.Logger,
+	daemonInst daemon.Daemon,
+	hostBootstrap bootstrap.Bootstrap,
+) (
+	bootstrap.Bootstrap, bool, error,
+) {
+
+	thisHost := hostBootstrap.ThisHost()
+
+	newHosts, err := daemonInst.GetGarageBootstrapHosts(ctx)
+	if err != nil {
+		return bootstrap.Bootstrap{}, false, fmt.Errorf("getting hosts from garage: %w", err)
+	}
+
+	// the daemon's view of this host's bootstrap info takes precedence over
+	// whatever is in garage
+	newHosts[thisHost.Name] = thisHost
+
+	newHostsHash, err := bootstrap.HostsHash(newHosts)
+	if err != nil {
+		return bootstrap.Bootstrap{}, false, fmt.Errorf("calculating hash of new hosts: %w", err)
+	}
+
+	currHostsHash, err := bootstrap.HostsHash(hostBootstrap.Hosts)
+	if err != nil {
+		return bootstrap.Bootstrap{}, false, fmt.Errorf("calculating hash of current hosts: %w", err)
+	}
+
+	if bytes.Equal(newHostsHash, currHostsHash) {
+		return hostBootstrap, false, nil
+	}
+
+	hostBootstrap.Hosts = newHosts
+
+	if err := writeBootstrapToStateDir(hostBootstrap); err != nil {
+		return bootstrap.Bootstrap{}, false, fmt.Errorf("writing new bootstrap to data dir: %w", err)
+	}
+
+	return hostBootstrap, true, nil
+}
+
+// runs a single pmux process of daemon, returning only once the env.Context has
+// been canceled or bootstrap info has been changed. This will always block
+// until the spawned pmux has returned, and returns a copy of hostBootstrap with
+// updated boostrap info.
+func runDaemonPmuxOnce(
+	ctx context.Context,
+	logger *mlog.Logger,
+	hostBootstrap bootstrap.Bootstrap,
+	daemonConfig daemon.Config,
+) (
+	bootstrap.Bootstrap, error,
+) {
+	daemonInst, err := daemon.New(
+		ctx,
+		logger.WithNamespace("daemon"),
+		daemonConfig,
+		hostBootstrap,
+		envRuntimeDirPath,
+		envBinDirPath,
+		envStateDirPath,
+		nil,
+	)
+	if err != nil {
+		return bootstrap.Bootstrap{}, fmt.Errorf("initializing daemon: %w", err)
+	}
+	defer func() {
+		// context.Background() is deliberate here. At this point the entire
+		// process is shutting down, so whatever owns the process should decide
+		// when it's been too long.
+		if err := daemonInst.Shutdown(context.Background()); err != nil {
+			logger.Error(ctx, "failed to cleanly shutdown daemon", err)
+		}
+	}()
+
+	{
+		logger := logger.WithNamespace("http")
+		httpSrv, err := newHTTPServer(
+			ctx, logger, daemon.NewRPC(daemonInst),
+		)
+		if err != nil {
+			return bootstrap.Bootstrap{}, fmt.Errorf("starting HTTP server: %w", err)
+		}
+		defer func() {
+			// see comment in daemonInst shutdown logic regarding background
+			// context.
+			if err := httpSrv.Shutdown(context.Background()); err != nil {
+				logger.Error(ctx, "Failed to cleanly shutdown http server", err)
+			}
+		}()
+	}
+
+	ticker := time.NewTicker(3 * time.Minute)
+	defer ticker.Stop()
+
+	for {
+		select {
+
+		case <-ctx.Done():
+			return bootstrap.Bootstrap{}, ctx.Err()
+
+		case <-ticker.C:
+
+			logger.Info(ctx, "checking for changes to bootstrap")
+
+			var (
+				changed bool
+				err     error
+			)
+
+			if hostBootstrap, changed, err = reloadBootstrap(
+				ctx, logger, daemonInst, hostBootstrap,
+			); err != nil {
+				return bootstrap.Bootstrap{}, fmt.Errorf("reloading bootstrap: %w", err)
+
+			} else if changed {
+				fmt.Fprintln(os.Stderr, "bootstrap info has changed, restarting all processes")
+				return hostBootstrap, nil
+			}
+		}
+	}
+}
+
 var subCmdDaemon = subCmd{
 	name:  "daemon",
 	descr: "Runs the isle daemon (Default if no sub-command given)",
@@ -65,7 +214,7 @@ var subCmdDaemon = subCmd{
 		defer runtimeDirCleanup()
 
 		var (
-			bootstrapStateDirPath = bootstrap.StateDirPath(daemonEnvVars.StateDirPath)
+			bootstrapStateDirPath = bootstrap.StateDirPath(envStateDirPath)
 			bootstrapAppDirPath   = bootstrap.AppDirPath(envAppDirPath)
 
 			hostBootstrapPath string
@@ -127,36 +276,16 @@ var subCmdDaemon = subCmd{
 			return fmt.Errorf("merging daemon config into bootstrap data: %w", err)
 		}
 
-		daemonInst := daemon.NewDaemonRestarter(
+		for {
-			logger, daemonConfig, envBinDirPath, hostBootstrap, nil,
-		)
-		defer func() {
-			logger.Info(ctx, "Stopping child processes")
-			if err := daemonInst.Shutdown(); err != nil {
-				logger.Error(ctx, "Shutting down daemon cleanly failed, there may be orphaned child processes", err)
-			}
-			logger.Info(ctx, "Child processes successfully stopped")
-		}()
 
-		{
+			hostBootstrap, err = runDaemonPmuxOnce(ctx, logger, hostBootstrap, daemonConfig)
-			logger := logger.WithNamespace("http")
-			httpSrv, err := newHTTPServer(
-				ctx, logger, daemon.NewRPC(daemonInst),
-			)
-			if err != nil {
-				return fmt.Errorf("starting HTTP server: %w", err)
-			}
-			defer func() {
-				// see comment in daemonInst shutdown logic regarding background
-				// context.
-				logger.Info(ctx, "Shutting down HTTP socket")
-				if err := httpSrv.Shutdown(context.Background()); err != nil {
-					logger.Error(ctx, "Failed to cleanly shutdown http server", err)
-				}
-			}()
-		}
 
-		<-ctx.Done()
+			if errors.Is(err, context.Canceled) {
 				return nil
+
+			} else if err != nil {
+				return fmt.Errorf("running pmux for daemon: %w", err)
+			}
+		}
 	},
 }

go/cmd/entrypoint/daemon_util.go

@@ -66,12 +66,10 @@ func newHTTPServer(
 ) (
 	*http.Server, error,
 ) {
-	l, err := net.Listen("unix", daemonEnvVars.HTTPSocketPath)
+	l, err := net.Listen("unix", envSocketPath)
 	if err != nil {
 		return nil, fmt.Errorf(
-			"failed to listen on socket %q: %w",
+			"failed to listen on socket %q: %w", envSocketPath, err,
-			daemonEnvVars.HTTPSocketPath,
-			err,
 		)
 	}
 

go/cmd/entrypoint/garage.go

@@ -13,7 +13,7 @@ import (
 // order to prevent it from doing so.
 func initMCConfigDir() (string, error) {
 	var (
-		path           = filepath.Join(daemonEnvVars.StateDirPath, "mc")
+		path           = filepath.Join(envStateDirPath, "mc")
 		sharePath      = filepath.Join(path, "share")
 		configJSONPath = filepath.Join(path, "config.json")
 	)

go/cmd/entrypoint/jigs.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"slices"
+	"strings"
+)
+
+func envOr(name string, fallback func() string) string {
+	if v := os.Getenv(name); v != "" {
+		return v
+	}
+	return fallback()
+}
+
+func firstExistingDir(paths ...string) (string, error) {
+	var errs []error
+	for _, path := range paths {
+		stat, err := os.Stat(path)
+		switch {
+		case errors.Is(err, fs.ErrExist):
+			continue
+		case err != nil:
+			errs = append(
+				errs, fmt.Errorf("checking if path %q exists: %w", path, err),
+			)
+		case !stat.IsDir():
+			errs = append(
+				errs, fmt.Errorf("path %q exists but is not a directory", path),
+			)
+		default:
+			return path, nil
+		}
+	}
+
+	err := fmt.Errorf(
+		"no directory found at any of the following paths: %s",
+		strings.Join(paths, ", "),
+	)
+	if len(errs) > 0 {
+		err = errors.Join(slices.Insert(errs, 0, err)...)
+	}
+	return "", err
+}

go/cmd/entrypoint/main.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"context"
-	"isle/daemon"
+	"fmt"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -10,8 +10,14 @@ import (
 
 	"dev.mediocregopher.com/mediocre-go-lib.git/mctx"
 	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
+	"github.com/adrg/xdg"
 )
 
+// The purpose of this binary is to act as the entrypoint of the isle
+// process. It processes the command-line arguments which are passed in, and
+// then passes execution along to an appropriate binary housed in AppDir/bin
+// (usually a bash script, which is more versatile than a go program).
+
 func getAppDirPath() string {
 	appDirPath := os.Getenv("APPDIR")
 	if appDirPath == "" {
@@ -20,10 +26,39 @@ func getAppDirPath() string {
 	return appDirPath
 }
 
+func getRPCSocketDirPath() string {
+	path, err := firstExistingDir(
+		"/run",
+		"/var/run",
+		"/tmp",
+		"/dev/shm",
+	)
+	if err != nil {
+		panic(fmt.Sprintf("Failed to find directory for RPC socket: %v", err))
+	}
+
+	return path
+}
+
+// RUNTIME_DIRECTORY/STATE_DIRECTORY are used by the systemd service in
+// conjunction with the RuntimeDirectory/StateDirectory directives.
 var (
-	daemonEnvVars = daemon.GetEnvVars()
 	envAppDirPath     = getAppDirPath()
+	envRuntimeDirPath = envOr(
+		"RUNTIME_DIRECTORY",
+		func() string { return filepath.Join(xdg.RuntimeDir, "isle") },
+	)
+	envStateDirPath = envOr(
+		"STATE_DIRECTORY",
+		func() string { return filepath.Join(xdg.StateHome, "isle") },
+	)
 	envBinDirPath = filepath.Join(envAppDirPath, "bin")
+	envSocketPath = envOr(
+		"ISLE_SOCKET_PATH",
+		func() string {
+			return filepath.Join(getRPCSocketDirPath(), "isle-daemon.sock")
+		},
+	)
 )
 
 func binPath(name string) string {

go/cmd/entrypoint/proc_lock.go

@@ -16,7 +16,7 @@ import (
 var errDaemonNotRunning = errors.New("no isle daemon process running")
 
 func lockFilePath() string {
-	return filepath.Join(daemonEnvVars.RuntimeDirPath, "lock")
+	return filepath.Join(envRuntimeDirPath, "lock")
 }
 
 func writeLock() error {
@@ -49,11 +49,11 @@ func writeLock() error {
 // returns a cleanup function which will clean up the created runtime directory.
 func setupAndLockRuntimeDir(ctx context.Context, logger *mlog.Logger) (func(), error) {
 
-	ctx = mctx.Annotate(ctx, "runtimeDirPath", daemonEnvVars.RuntimeDirPath)
+	ctx = mctx.Annotate(ctx, "runtimeDirPath", envRuntimeDirPath)
 	logger.Info(ctx, "will use runtime directory for temporary state")
 
-	if err := os.MkdirAll(daemonEnvVars.RuntimeDirPath, 0700); err != nil {
+	if err := os.MkdirAll(envRuntimeDirPath, 0700); err != nil {
-		return nil, fmt.Errorf("creating directory %q: %w", daemonEnvVars.RuntimeDirPath, err)
+		return nil, fmt.Errorf("creating directory %q: %w", envRuntimeDirPath, err)
 
 	} else if err := writeLock(); err != nil {
 		return nil, err
@@ -61,7 +61,7 @@ func setupAndLockRuntimeDir(ctx context.Context, logger *mlog.Logger) (func(), e
 
 	return func() {
 		logger.Info(ctx, "cleaning up runtime directory")
-		if err := os.RemoveAll(daemonEnvVars.RuntimeDirPath); err != nil {
+		if err := os.RemoveAll(envRuntimeDirPath); err != nil {
 			logger.Error(ctx, "removing temporary directory", err)
 		}
 	}, nil

go/cmd/entrypoint/sub_cmd.go

@@ -110,7 +110,7 @@ func (ctx subCmdCtx) doSubCmd(subCmds ...subCmd) error {
 	}
 
 	daemonRCPClient := jsonrpc2.NewUnixHTTPClient(
-		daemonEnvVars.HTTPSocketPath, daemonHTTPRPCPath,
+		envSocketPath, daemonHTTPRPCPath,
 	)
 
 	err := subCmd.do(subCmdCtx{

go/daemon/bootstrap.go

@@ -1,50 +0,0 @@
-package daemon
-
-import (
-	"errors"
-	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-
-	"isle/bootstrap"
-)
-
-func loadHostBootstrap(stateDirPath string) (bootstrap.Bootstrap, error) {
-	path := bootstrap.StateDirPath(stateDirPath)
-
-	hostBootstrap, err := bootstrap.FromFile(path)
-	if errors.Is(err, fs.ErrNotExist) {
-		return bootstrap.Bootstrap{}, fmt.Errorf(
-			"%q not found, has the daemon ever been run?",
-			stateDirPath,
-		)
-
-	} else if err != nil {
-		return bootstrap.Bootstrap{}, fmt.Errorf("loading %q: %w", stateDirPath, err)
-	}
-
-	return hostBootstrap, nil
-}
-
-func writeBootstrapToStateDir(
-	stateDirPath string, hostBootstrap bootstrap.Bootstrap,
-) error {
-	var (
-		path    = bootstrap.StateDirPath(stateDirPath)
-		dirPath = filepath.Dir(path)
-	)
-
-	if err := os.MkdirAll(dirPath, 0700); err != nil {
-		return fmt.Errorf("creating directory %q: %w", dirPath, err)
-	}
-
-	f, err := os.Create(path)
-	if err != nil {
-		return fmt.Errorf("creating file %q: %w", path, err)
-	}
-
-	defer f.Close()
-
-	return hostBootstrap.WriteTo(f)
-}

go/daemon/child_pmux.go

@@ -1,95 +0,0 @@
-package daemon
-
-import (
-	"context"
-	"fmt"
-
-	"code.betamike.com/micropelago/pmux/pmuxlib"
-)
-
-func (d *daemon) newPmuxConfig() (pmuxlib.Config, error) {
-	nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(
-		d.opts.EnvVars.RuntimeDirPath,
-		d.binDirPath,
-		d.hostBootstrap,
-		d.config,
-	)
-	if err != nil {
-		return pmuxlib.Config{}, fmt.Errorf("generating nebula config: %w", err)
-	}
-
-	dnsmasqPmuxProcConfig, err := dnsmasqPmuxProcConfig(
-		d.opts.EnvVars.RuntimeDirPath,
-		d.binDirPath,
-		d.hostBootstrap,
-		d.config,
-	)
-	if err != nil {
-		return pmuxlib.Config{}, fmt.Errorf(
-			"generating dnsmasq config: %w", err,
-		)
-	}
-
-	garagePmuxProcConfigs, err := garagePmuxProcConfigs(
-		d.opts.EnvVars.RuntimeDirPath, d.binDirPath, d.hostBootstrap, d.config,
-	)
-	if err != nil {
-		return pmuxlib.Config{}, fmt.Errorf(
-			"generating garage children configs: %w", err,
-		)
-	}
-
-	return pmuxlib.Config{
-		Processes: append(
-			[]pmuxlib.ProcessConfig{
-				nebulaPmuxProcConfig,
-				dnsmasqPmuxProcConfig,
-			},
-			garagePmuxProcConfigs...,
-		),
-	}, nil
-}
-
-func (d *daemon) postPmuxInit(ctx context.Context) error {
-	d.logger.Info(ctx, "waiting for nebula VPN to come online")
-	if err := waitForNebula(ctx, d.hostBootstrap); err != nil {
-		return fmt.Errorf("waiting for nebula to start: %w", err)
-	}
-
-	d.logger.Info(ctx, "waiting for garage instances to come online")
-	if err := d.waitForGarage(ctx); err != nil {
-		return fmt.Errorf("waiting for garage to start: %w", err)
-	}
-
-	if len(d.config.Storage.Allocations) > 0 {
-
-		err := until(ctx, func(ctx context.Context) error {
-			err := garageApplyLayout(ctx, d.logger, d.hostBootstrap, d.config)
-			if err != nil {
-				d.logger.Error(ctx, "applying garage layout", err)
-				return err
-			}
-
-			return nil
-		})
-
-		if err != nil {
-			return fmt.Errorf("applying garage layout: %w", err)
-		}
-	}
-
-	if !d.opts.SkipHostBootstrapPush {
-		if err := until(ctx, func(ctx context.Context) error {
-			if err := d.putGarageBoostrapHost(ctx); err != nil {
-				d.logger.Error(ctx, "updating host info in garage", err)
-				return err
-			}
-
-			return nil
-		}); err != nil {
-			return fmt.Errorf("updating host info in garage: %w", err)
-		}
-	}
-
-	return nil
-}

go/daemon/daemon.go

@@ -8,11 +8,23 @@ import (
 	"io"
 	"isle/bootstrap"
 	"os"
+	"time"
 
 	"code.betamike.com/micropelago/pmux/pmuxlib"
 	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
 )
 
+type daemon struct {
+	logger        *mlog.Logger
+	config        Config
+	hostBootstrap bootstrap.Bootstrap
+	stateDirPath  string
+	opts          Opts
+
+	pmuxCancelFn  context.CancelFunc
+	pmuxStoppedCh chan struct{}
+}
+
 // Daemon presents all functionality required for client frontends to interact
 // with isle, typically via the unix socket.
 type Daemon interface {
@@ -26,11 +38,12 @@ type Daemon interface {
 	)
 
 	// Shutdown blocks until all resources held or created by the daemon,
-	// including child processes it has started, have been cleaned up.
+	// including child processes it has started, have been cleaned up, or until
+	// the context is canceled.
 	//
 	// If this returns an error then it's possible that child processes are
 	// still running and are no longer managed.
-	Shutdown() error
+	Shutdown(context.Context) error
 }
 
 // Opts are optional parameters which can be passed in when initializing a new
@@ -43,8 +56,6 @@ type Opts struct {
 	// Stdout and Stderr are what the associated outputs from child processes
 	// will be directed to.
 	Stdout, Stderr io.Writer
-
-	EnvVars EnvVars // Defaults to that returned by GetEnvVars.
 }
 
 func (o *Opts) withDefaults() *Opts {
@@ -60,24 +71,9 @@ func (o *Opts) withDefaults() *Opts {
 		o.Stderr = os.Stderr
 	}
 
-	if o.EnvVars == (EnvVars{}) {
-		o.EnvVars = GetEnvVars()
-	}
-
 	return o
 }
 
-type daemon struct {
-	logger        *mlog.Logger
-	config        Config
-	hostBootstrap bootstrap.Bootstrap
-	binDirPath    string
-	opts          Opts
-
-	pmuxCancelFn  context.CancelFunc
-	pmuxStoppedCh chan struct{}
-}
-
 // New initialized and returns a Daemon. If initialization fails an error is
 // returned.
 func New(
@@ -85,39 +81,66 @@ func New(
 	logger *mlog.Logger,
 	config Config,
 	hostBootstrap bootstrap.Bootstrap,
-	binDirPath string,
+	runtimeDirPath, binDirPath, stateDirPath string,
 	opts *Opts,
 ) (
 	Daemon, error,
 ) {
 	opts = opts.withDefaults()
 
+	nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(
+		runtimeDirPath, binDirPath, hostBootstrap, config,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("generating nebula config: %w", err)
+	}
+
+	dnsmasqPmuxProcConfig, err := dnsmasqPmuxProcConfig(
+		runtimeDirPath, binDirPath, hostBootstrap, config,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("generating dnsmasq config: %w", err)
+	}
+
+	garagePmuxProcConfigs, err := garagePmuxProcConfigs(
+		runtimeDirPath, binDirPath, hostBootstrap, config,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("generating garage children configs: %w", err)
+	}
+
+	pmuxConfig := pmuxlib.Config{
+		Processes: append(
+			[]pmuxlib.ProcessConfig{
+				nebulaPmuxProcConfig,
+				dnsmasqPmuxProcConfig,
+			},
+			garagePmuxProcConfigs...,
+		),
+	}
+
 	pmuxCtx, pmuxCancelFn := context.WithCancel(context.Background())
 
 	d := &daemon{
 		logger:        logger,
 		config:        config,
 		hostBootstrap: hostBootstrap,
-		binDirPath:    binDirPath,
+		stateDirPath:  stateDirPath,
 		opts:          *opts,
 		pmuxCancelFn:  pmuxCancelFn,
 		pmuxStoppedCh: make(chan struct{}),
 	}
 
-	pmuxConfig, err := d.newPmuxConfig()
-	if err != nil {
-		return nil, fmt.Errorf("generating pmux config: %w", err)
-	}
-
 	go func() {
 		defer close(d.pmuxStoppedCh)
 		pmuxlib.Run(pmuxCtx, d.opts.Stdout, d.opts.Stderr, pmuxConfig)
-		d.logger.Debug(pmuxCtx, "pmux stopped")
 	}()
 
 	if initErr := d.postPmuxInit(ctx); initErr != nil {
 		logger.Warn(ctx, "failed to initialize daemon, shutting down child processes", err)
-		if err := d.Shutdown(); err != nil {
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+		defer cancel()
+		if err := d.Shutdown(shutdownCtx); err != nil {
 			panic(fmt.Sprintf(
 				"failed to shut down child processes after initialization"+
 					" error, there may be zombie children leftover."+
@@ -132,8 +155,56 @@ func New(
 	return d, nil
 }
 
-func (d *daemon) Shutdown() error {
+func (d *daemon) postPmuxInit(ctx context.Context) error {
-	d.pmuxCancelFn()
+	d.logger.Info(ctx, "waiting for nebula VPN to come online")
-	<-d.pmuxStoppedCh
+	if err := waitForNebula(ctx, d.hostBootstrap); err != nil {
+		return fmt.Errorf("waiting for nebula to start: %w", err)
+	}
+
+	d.logger.Info(ctx, "waiting for garage instances to come online")
+	if err := d.waitForGarage(ctx); err != nil {
+		return fmt.Errorf("waiting for garage to start: %w", err)
+	}
+
+	if len(d.config.Storage.Allocations) > 0 {
+
+		err := until(ctx, func(ctx context.Context) error {
+			err := garageApplyLayout(ctx, d.logger, d.hostBootstrap, d.config)
+			if err != nil {
+				d.logger.Error(ctx, "applying garage layout", err)
+				return err
+			}
+
+			return nil
+		})
+
+		if err != nil {
+			return fmt.Errorf("applying garage layout: %w", err)
+		}
+	}
+
+	if !d.opts.SkipHostBootstrapPush {
+		if err := until(ctx, func(ctx context.Context) error {
+			if err := d.putGarageBoostrapHost(ctx); err != nil {
+				d.logger.Error(ctx, "updating host info in garage", err)
+				return err
+			}
+
+			return nil
+		}); err != nil {
+			return fmt.Errorf("updating host info in garage: %w", err)
+		}
+	}
+
 	return nil
 }
+
+func (d *daemon) Shutdown(ctx context.Context) error {
+	d.pmuxCancelFn()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-d.pmuxStoppedCh:
+		return nil
+	}
+}

go/daemon/daemon_restarter.go

@@ -1,262 +0,0 @@
-package daemon
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"fmt"
-	"isle/bootstrap"
-	"sync"
-	"time"
-
-	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
-)
-
-const (
-	daemonRestarterStateInitializing = iota
-	daemonRestarterStateOk
-	daemonRestarterStateRestarting
-	daemonRestarterStateShutdown
-)
-
-type daemonRestarter struct {
-	logger        *mlog.Logger
-	daemonConfig  Config
-	envBinDirPath string
-	opts          *Opts
-
-	l             sync.Mutex
-	state         int
-	inner         Daemon
-	currBootstrap bootstrap.Bootstrap
-
-	cancelFn  context.CancelFunc
-	stoppedCh chan struct{}
-}
-
-// NewDaemonRestarter will wrap a Daemon such that it will be automatically
-// shutdown and re-created whenever there's changes in the cluster which require
-// the configuration to be changed (e.g. a new nebula lighthouse).
-//
-// While still starting up the daemon for the first time all methods will return
-// ErrInitializing, except Shutdown which will block until initialization is
-// canceled.
-//
-// During a restart all methods will return ErrRestarting, except Shutdown which
-// will block until the currently executing restart is finished and then
-// shutdown cleanly from there.
-//
-// TODO make daemonRestarter smarter, it currently restarts on _any_ change, but
-// it should restart itself only when there's something actually requiring a
-// restart.
-func NewDaemonRestarter(
-	logger *mlog.Logger,
-	daemonConfig Config,
-	envBinDirPath string,
-	currBootstrap bootstrap.Bootstrap,
-	opts *Opts,
-) Daemon {
-	ctx, cancelFn := context.WithCancel(context.Background())
-
-	dr := &daemonRestarter{
-		logger:        logger,
-		daemonConfig:  daemonConfig,
-		envBinDirPath: envBinDirPath,
-		opts:          opts.withDefaults(),
-		currBootstrap: currBootstrap,
-		cancelFn:      cancelFn,
-		stoppedCh:     make(chan struct{}),
-	}
-
-	go func() {
-		dr.restartLoop(ctx)
-		dr.logger.Debug(ctx, "DaemonRestarter stopped")
-		close(dr.stoppedCh)
-	}()
-
-	return dr
-}
-
-func withInnerDaemon[Res any](
-	dr *daemonRestarter, fn func(Daemon) (Res, error),
-) (Res, error) {
-	var zero Res
-	dr.l.Lock()
-	inner, state := dr.inner, dr.state
-	dr.l.Unlock()
-
-	switch state {
-	case daemonRestarterStateInitializing:
-		return zero, ErrInitializing
-	case daemonRestarterStateOk:
-		return fn(inner)
-	case daemonRestarterStateRestarting:
-		return zero, ErrRestarting
-	case daemonRestarterStateShutdown:
-		return zero, errors.New("already shutdown")
-	default:
-		panic(fmt.Sprintf("unknown state %d", dr.state))
-	}
-}
-
-// creates a new bootstrap file using available information from the network. If
-// the new bootstrap file is different than the existing one, the existing one
-// is overwritten and true is returned.
-func (dr *daemonRestarter) checkBootstrap(
-	ctx context.Context, hostBootstrap bootstrap.Bootstrap,
-) (
-	bootstrap.Bootstrap, bool, error,
-) {
-
-	thisHost := hostBootstrap.ThisHost()
-
-	newHosts, err := dr.inner.GetGarageBootstrapHosts(ctx)
-	if err != nil {
-		return bootstrap.Bootstrap{}, false, fmt.Errorf("getting hosts from garage: %w", err)
-	}
-
-	// the daemon's view of this host's bootstrap info takes precedence over
-	// whatever is in garage
-	newHosts[thisHost.Name] = thisHost
-
-	newHostsHash, err := bootstrap.HostsHash(newHosts)
-	if err != nil {
-		return bootstrap.Bootstrap{}, false, fmt.Errorf("calculating hash of new hosts: %w", err)
-	}
-
-	currHostsHash, err := bootstrap.HostsHash(hostBootstrap.Hosts)
-	if err != nil {
-		return bootstrap.Bootstrap{}, false, fmt.Errorf("calculating hash of current hosts: %w", err)
-	}
-
-	if bytes.Equal(newHostsHash, currHostsHash) {
-		return hostBootstrap, false, nil
-	}
-
-	hostBootstrap.Hosts = newHosts
-
-	return hostBootstrap, true, nil
-}
-
-// blocks until a new bootstrap is available or context is canceled
-func (dr *daemonRestarter) watchForChanges(ctx context.Context) bootstrap.Bootstrap {
-
-	ticker := time.NewTicker(3 * time.Minute)
-	defer ticker.Stop()
-
-	for {
-		select {
-
-		case <-ctx.Done():
-			return bootstrap.Bootstrap{}
-
-		case <-ticker.C:
-
-			dr.logger.Info(ctx, "Checking for changes to bootstrap")
-
-			newBootstrap, changed, err := dr.checkBootstrap(
-				ctx, dr.currBootstrap,
-			)
-			if err != nil {
-				dr.logger.Error(ctx, "Checking bootstrap for changes failed", err)
-				continue
-			} else if !changed {
-				continue
-			}
-
-			err = writeBootstrapToStateDir(dr.opts.EnvVars.StateDirPath, newBootstrap)
-			if err != nil {
-				dr.logger.Error(ctx, "Writing new bootstrap to disk failed", err)
-				continue
-			}
-
-			return newBootstrap
-		}
-	}
-}
-
-func (dr *daemonRestarter) restartLoop(ctx context.Context) {
-	defer func() {
-		dr.l.Lock()
-		dr.state = daemonRestarterStateShutdown
-		inner := dr.inner
-		dr.l.Unlock()
-
-		if inner != nil {
-			if err := inner.Shutdown(); err != nil {
-				dr.logger.Fatal(ctx, "failed to cleanly shutdown daemon, there may be orphaned child processes", err)
-			}
-		}
-	}()
-
-	wait := func(d time.Duration) bool {
-		select {
-		case <-ctx.Done():
-			return false
-		case <-time.After(d):
-			return true
-		}
-	}
-
-	for {
-		if ctx.Err() != nil {
-			return
-		}
-
-		dr.logger.Info(ctx, "Creating new daemon")
-		daemonInst, err := New(
-			ctx,
-			dr.logger.WithNamespace("daemon"),
-			dr.daemonConfig,
-			dr.currBootstrap,
-			dr.envBinDirPath,
-			dr.opts,
-		)
-		if errors.Is(err, context.Canceled) {
-			return
-		} else if err != nil {
-			dr.logger.Error(ctx, "failed to initialize daemon", err)
-			if !wait(1 * time.Second) {
-				return
-			}
-			continue
-		}
-
-		dr.l.Lock()
-		dr.inner = daemonInst
-		dr.state = daemonRestarterStateOk
-		dr.l.Unlock()
-
-		newBootstrap := dr.watchForChanges(ctx)
-		if ctx.Err() != nil {
-			return
-		}
-
-		dr.logger.Info(ctx, "Bootstrap has changed, will restart daemon")
-		dr.l.Lock()
-		dr.currBootstrap = newBootstrap
-		dr.state = daemonRestarterStateRestarting
-		dr.l.Unlock()
-
-		dr.logger.Info(ctx, "Shutting down previous daemon")
-		if err := dr.inner.Shutdown(); err != nil {
-			dr.logger.Fatal(ctx, "failed to cleanly shutdown daemon, there may be orphaned child processes", err)
-		}
-	}
-}
-
-func (dr *daemonRestarter) GetGarageBootstrapHosts(
-	ctx context.Context,
-) (
-	map[string]bootstrap.Host, error,
-) {
-	return withInnerDaemon(dr, func(inner Daemon) (map[string]bootstrap.Host, error) {
-		return inner.GetGarageBootstrapHosts(ctx)
-	})
-}
-
-func (dr *daemonRestarter) Shutdown() error {
-	dr.cancelFn()
-	<-dr.stoppedCh
-	return nil
-}

go/daemon/env.go

@@ -1,102 +0,0 @@
-package daemon
-
-import (
-	"errors"
-	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"slices"
-	"strings"
-	"sync"
-
-	"github.com/adrg/xdg"
-)
-
-// EnvVars are variables which are derived based on the environment which the
-// process is running in.
-type EnvVars struct {
-	RuntimeDirPath string // TODO should be private to this package
-	StateDirPath   string // TODO should be private to this package
-	HTTPSocketPath string
-}
-
-func getRPCSocketDirPath() string {
-	path, err := firstExistingDir(
-		"/run",
-		"/var/run",
-		"/tmp",
-		"/dev/shm",
-	)
-	if err != nil {
-		panic(fmt.Sprintf("Failed to find directory for RPC socket: %v", err))
-	}
-
-	return path
-}
-
-// GetEnvVars will return the EnvVars of the current processes, as determined by
-// the process's environment.
-var GetEnvVars = sync.OnceValue(func() (v EnvVars) {
-	// RUNTIME_DIRECTORY/STATE_DIRECTORY are used by the systemd service in
-	// conjunction with the RuntimeDirectory/StateDirectory directives.
-
-	v.RuntimeDirPath = envOr(
-		"RUNTIME_DIRECTORY",
-		func() string { return filepath.Join(xdg.RuntimeDir, "isle") },
-	)
-
-	v.StateDirPath = envOr(
-		"STATE_DIRECTORY",
-		func() string { return filepath.Join(xdg.StateHome, "isle") },
-	)
-
-	v.HTTPSocketPath = envOr(
-		"ISLE_SOCKET_PATH",
-		func() string {
-			return filepath.Join(getRPCSocketDirPath(), "isle-daemon.sock")
-		},
-	)
-
-	return
-})
-
-////////////////////////////////////////////////////////////////////////////////
-// Jigs
-
-func envOr(name string, fallback func() string) string {
-	if v := os.Getenv(name); v != "" {
-		return v
-	}
-	return fallback()
-}
-
-func firstExistingDir(paths ...string) (string, error) {
-	var errs []error
-	for _, path := range paths {
-		stat, err := os.Stat(path)
-		switch {
-		case errors.Is(err, fs.ErrExist):
-			continue
-		case err != nil:
-			errs = append(
-				errs, fmt.Errorf("checking if path %q exists: %w", path, err),
-			)
-		case !stat.IsDir():
-			errs = append(
-				errs, fmt.Errorf("path %q exists but is not a directory", path),
-			)
-		default:
-			return path, nil
-		}
-	}
-
-	err := fmt.Errorf(
-		"no directory found at any of the following paths: %s",
-		strings.Join(paths, ", "),
-	)
-	if len(errs) > 0 {
-		err = errors.Join(slices.Insert(errs, 0, err)...)
-	}
-	return "", err
-}

go/daemon/errors.go

@@ -1,13 +0,0 @@
-package daemon
-
-import "isle/daemon/jsonrpc2"
-
-var (
-	// ErrInitializing is returned when a cluster is unavailable due to still
-	// being initialized.
-	ErrInitializing = jsonrpc2.NewError(1, "Cluster is being initialized")
-
-	// ErrRestarting is returned when a cluster is unavailable due to being
-	// restarted.
-	ErrRestarting = jsonrpc2.NewError(2, "Cluster is being restarted")
-)

tests/utils/with-1-data-1-empty-node-cluster.sh

@@ -58,7 +58,7 @@ EOF
         --name        "testing" \
         > admin.json
 
-    isle daemon -l debug --config-path daemon.yml >daemon.log 2>&1 &
+    isle daemon --config-path daemon.yml >daemon.log 2>&1 &
     pid="$!"
     echo "Waiting for primus daemon (process $pid) to initialize"
 
@@ -82,7 +82,7 @@ EOF
             device: isle-secondus
 EOF
 
-        isle daemon -l debug -c daemon.yml -b "$secondus_bootstrap" >daemon.log 2>&1 &
+        isle daemon -c daemon.yml -b "$secondus_bootstrap" >daemon.log 2>&1 &
         pid="$!"
         echo "Waiting for secondus daemon (process $!) to initialize"