isle/go/cmd/entrypoint/daemon.go

package main

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"time"

	"isle/bootstrap"
	"isle/daemon"

	"dev.mediocregopher.com/mediocre-go-lib.git/mctx"
	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
)

// The daemon sub-command deals with starting an actual isle daemon
// process, which is required to be running for most other Isle
// functionality. The sub-command does the following:
//
// * Creates and locks the runtime directory.
//
// * Creates the data directory and copies the appdir bootstrap file into there,
// if it's not already there.
//
// * Merges daemon configuration into the bootstrap configuration, and rewrites
// the bootstrap file.
//
// * Sets up environment variables that all other sub-processes then use, based
// on the runtime dir.
//
// * Dynamically creates the root pmux config and runs pmux.
//
// * (On exit) cleans up the runtime directory.

// creates a new bootstrap file using available information from the network. If
// the new bootstrap file is different than the existing one, the existing one
// is overwritten and true is returned.
func reloadBootstrap(
	ctx context.Context,
	logger *mlog.Logger,
	daemonInst daemon.Daemon,
	hostBootstrap bootstrap.Bootstrap,
) (
	bootstrap.Bootstrap, bool, error,
) {

	thisHost := hostBootstrap.ThisHost()

	newHosts, err := daemonInst.GetGarageBootstrapHosts(ctx)
	if err != nil {
		return bootstrap.Bootstrap{}, false, fmt.Errorf("getting hosts from garage: %w", err)
	}

	// the daemon's view of this host's bootstrap info takes precedence over
	// whatever is in garage
	newHosts[thisHost.Name] = thisHost

	newHostsHash, err := bootstrap.HostsHash(newHosts)
	if err != nil {
		return bootstrap.Bootstrap{}, false, fmt.Errorf("calculating hash of new hosts: %w", err)
	}

	currHostsHash, err := bootstrap.HostsHash(hostBootstrap.Hosts)
	if err != nil {
		return bootstrap.Bootstrap{}, false, fmt.Errorf("calculating hash of current hosts: %w", err)
	}

	if bytes.Equal(newHostsHash, currHostsHash) {
		return hostBootstrap, false, nil
	}

	hostBootstrap.Hosts = newHosts

	if err := writeBootstrapToStateDir(hostBootstrap); err != nil {
		return bootstrap.Bootstrap{}, false, fmt.Errorf("writing new bootstrap to data dir: %w", err)
	}

	return hostBootstrap, true, nil
}

// runs a single pmux process of daemon, returning only once the env.Context has
// been canceled or bootstrap info has been changed. This will always block
// until the spawned pmux has returned, and returns a copy of hostBootstrap with
// updated boostrap info.
func runDaemonPmuxOnce(
	ctx context.Context,
	logger *mlog.Logger,
	hostBootstrap bootstrap.Bootstrap,
	daemonConfig daemon.Config,
) (
	bootstrap.Bootstrap, error,
) {
	daemonInst, err := daemon.New(
		ctx,
		logger.WithNamespace("daemon"),
		daemonConfig,
		hostBootstrap,
		envRuntimeDirPath,
		envBinDirPath,
		envStateDirPath,
		nil,
	)
	if err != nil {
		return bootstrap.Bootstrap{}, fmt.Errorf("initializing daemon: %w", err)
	}
	defer func() {
		// context.Background() is deliberate here. At this point the entire
		// process is shutting down, so whatever owns the process should decide
		// when it's been too long.
		if err := daemonInst.Shutdown(context.Background()); err != nil {
			logger.Error(ctx, "failed to cleanly shutdown daemon", err)
		}
	}()

	ticker := time.NewTicker(3 * time.Minute)
	defer ticker.Stop()

	for {
		select {

		case <-ctx.Done():
			return bootstrap.Bootstrap{}, ctx.Err()

		case <-ticker.C:

			logger.Info(ctx, "checking for changes to bootstrap")

			var (
				changed bool
				err     error
			)

			if hostBootstrap, changed, err = reloadBootstrap(
				ctx, logger, daemonInst, hostBootstrap,
			); err != nil {
				return bootstrap.Bootstrap{}, fmt.Errorf("reloading bootstrap: %w", err)

			} else if changed {
				fmt.Fprintln(os.Stderr, "bootstrap info has changed, restarting all processes")
				return hostBootstrap, nil
			}
		}
	}
}

var subCmdDaemon = subCmd{
	name:  "daemon",
	descr: "Runs the isle daemon (Default if no sub-command given)",
	do: func(subCmdCtx subCmdCtx) error {

		flags := subCmdCtx.flagSet(false)

		daemonConfigPath := flags.StringP(
			"config-path", "c", "",
			"Optional path to a daemon.yml file to load configuration from.",
		)

		dumpConfig := flags.Bool(
			"dump-config", false,
			"Write the default configuration file to stdout and exit.",
		)

		bootstrapPath := flags.StringP(
			"bootstrap-path", "b", "",
			`Path to a bootstrap.json file. This only needs to be provided the first time the daemon is started, after that it is ignored. If the isle binary has a bootstrap built into it then this argument is always optional.`,
		)

		logLevelStr := flags.StringP(
			"log-level", "l", "info",
			`Maximum log level which should be output. Values can be "debug", "info", "warn", "error", "fatal". Does not apply to sub-processes`,
		)

		if err := flags.Parse(subCmdCtx.args); err != nil {
			return fmt.Errorf("parsing flags: %w", err)
		}

		ctx := subCmdCtx.ctx

		if *dumpConfig {
			return daemon.CopyDefaultConfig(os.Stdout, envAppDirPath)
		}

		logLevel := mlog.LevelFromString(*logLevelStr)
		if logLevel == nil {
			return fmt.Errorf("couldn't parse log level %q", *logLevelStr)
		}

		logger := subCmdCtx.logger.WithMaxLevel(logLevel.Int())

		runtimeDirCleanup, err := setupAndLockRuntimeDir(ctx, logger)
		if err != nil {
			return fmt.Errorf("setting up runtime directory: %w", err)
		}
		defer runtimeDirCleanup()

		var (
			bootstrapStateDirPath = bootstrap.StateDirPath(envStateDirPath)
			bootstrapAppDirPath   = bootstrap.AppDirPath(envAppDirPath)

			hostBootstrapPath string
			hostBootstrap     bootstrap.Bootstrap
		)

		tryLoadBootstrap := func(path string) bool {
			ctx := mctx.Annotate(ctx, "bootstrapFilePath", path)

			if err != nil {
				return false

			} else if hostBootstrap, err = bootstrap.FromFile(path); errors.Is(err, fs.ErrNotExist) {
				logger.WarnString(ctx, "bootstrap file not found")
				err = nil
				return false

			} else if err != nil {
				err = fmt.Errorf("parsing bootstrap.json at %q: %w", path, err)
				return false
			}

			logger.Info(ctx, "bootstrap file found")

			hostBootstrapPath = path
			return true
		}

		switch {
		case tryLoadBootstrap(bootstrapStateDirPath):
		case *bootstrapPath != "" && tryLoadBootstrap(*bootstrapPath):
		case tryLoadBootstrap(bootstrapAppDirPath):
		case err != nil:
			return fmt.Errorf("attempting to load bootstrap.json file: %w", err)
		default:
			return errors.New("No bootstrap.json file could be found, and one is not provided with --bootstrap-path")
		}

		if hostBootstrapPath != bootstrapStateDirPath {

			// If the bootstrap file is not being stored in the data dir, copy
			// it there, so it can be loaded from there next time.
			if err := writeBootstrapToStateDir(hostBootstrap); err != nil {
				return fmt.Errorf("writing bootstrap.json to data dir: %w", err)
			}
		}

		daemonConfig, err := daemon.LoadConfig(envAppDirPath, *daemonConfigPath)
		if err != nil {
			return fmt.Errorf("loading daemon config: %w", err)
		}

		// we update this Host's data using whatever configuration has been
		// provided by the daemon config. This way the daemon has the most
		// up-to-date possible bootstrap. This updated bootstrap will later get
		// updated in garage as a background daemon task, so other hosts will
		// see it as well.
		if hostBootstrap, err = coalesceDaemonConfigAndBootstrap(hostBootstrap, daemonConfig); err != nil {
			return fmt.Errorf("merging daemon config into bootstrap data: %w", err)
		}

		for {

			hostBootstrap, err = runDaemonPmuxOnce(ctx, logger, hostBootstrap, daemonConfig)

			if errors.Is(err, context.Canceled) {
				return nil

			} else if err != nil {
				return fmt.Errorf("running pmux for daemon: %w", err)
			}
		}
	},
}