isle/go/cmd/entrypoint/daemon.go

package main

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"sync"
	"time"

	"isle/bootstrap"
	"isle/daemon"

	"code.betamike.com/micropelago/pmux/pmuxlib"
	"github.com/mediocregopher/mediocre-go-lib/v2/mctx"
	"github.com/mediocregopher/mediocre-go-lib/v2/mlog"
)

// The daemon sub-command deals with starting an actual isle daemon
// process, which is required to be running for most other Isle
// functionality. The sub-command does the following:
//
// * Creates and locks the runtime directory.
//
// * Creates the data directory and copies the appdir bootstrap file into there,
// if it's not already there.
//
// * Merges daemon configuration into the bootstrap configuration, and rewrites
// the bootstrap file.
//
// * Sets up environment variables that all other sub-processes then use, based
// on the runtime dir.
//
// * Dynamically creates the root pmux config and runs pmux.
//
// * (On exit) cleans up the runtime directory.

// creates a new bootstrap file using available information from the network. If
// the new bootstrap file is different than the existing one, the existing one
// is overwritten and true is returned.
func reloadBootstrap(
	ctx context.Context,
	logger *mlog.Logger,
	hostBootstrap bootstrap.Bootstrap,
) (
	bootstrap.Bootstrap, bool, error,
) {

	thisHost := hostBootstrap.ThisHost()

	newHosts, err := hostBootstrap.GetGarageBootstrapHosts(ctx, logger)
	if err != nil {
		return bootstrap.Bootstrap{}, false, fmt.Errorf("getting hosts from garage: %w", err)
	}

	// the daemon's view of this host's bootstrap info takes precedence over
	// whatever is in garage
	newHosts[thisHost.Name] = thisHost

	newHostsHash, err := bootstrap.HostsHash(newHosts)
	if err != nil {
		return bootstrap.Bootstrap{}, false, fmt.Errorf("calculating hash of new hosts: %w", err)
	}

	currHostsHash, err := bootstrap.HostsHash(hostBootstrap.Hosts)
	if err != nil {
		return bootstrap.Bootstrap{}, false, fmt.Errorf("calculating hash of current hosts: %w", err)
	}

	if bytes.Equal(newHostsHash, currHostsHash) {
		return hostBootstrap, false, nil
	}

	hostBootstrap.Hosts = newHosts

	if err := writeBootstrapToDataDir(hostBootstrap); err != nil {
		return bootstrap.Bootstrap{}, false, fmt.Errorf("writing new bootstrap to data dir: %w", err)
	}

	return hostBootstrap, true, nil
}

// runs a single pmux process of daemon, returning only once the env.Context has
// been canceled or bootstrap info has been changed. This will always block
// until the spawned pmux has returned, and returns a copy of hostBootstrap with
// updated boostrap info.
func runDaemonPmuxOnce(
	ctx context.Context,
	logger *mlog.Logger,
	hostBootstrap bootstrap.Bootstrap,
	daemonConfig daemon.Config,
) (
	bootstrap.Bootstrap, error,
) {

	nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(hostBootstrap, daemonConfig)
	if err != nil {
		return bootstrap.Bootstrap{}, fmt.Errorf("generating nebula config: %w", err)
	}

	dnsmasqPmuxProcConfig, err := dnsmasqPmuxProcConfig(hostBootstrap, daemonConfig)
	if err != nil {
		return bootstrap.Bootstrap{}, fmt.Errorf("generating dnsmasq config: %w", err)
	}

	garagePmuxProcConfigs, err := garagePmuxProcConfigs(hostBootstrap, daemonConfig)
	if err != nil {
		return bootstrap.Bootstrap{}, fmt.Errorf("generating garage children configs: %w", err)
	}

	pmuxConfig := pmuxlib.Config{
		Processes: append(
			[]pmuxlib.ProcessConfig{
				nebulaPmuxProcConfig,
				dnsmasqPmuxProcConfig,
			},
			garagePmuxProcConfigs...,
		),
	}

	var wg sync.WaitGroup
	defer wg.Wait()

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	wg.Add(1)
	go func() {
		defer wg.Done()
		pmuxlib.Run(ctx, os.Stdout, os.Stderr, pmuxConfig)
	}()

	if err := waitForGarageAndNebula(ctx, logger, hostBootstrap, daemonConfig); err != nil {
		return bootstrap.Bootstrap{}, fmt.Errorf("waiting for nebula/garage to start up: %w", err)
	}

	if len(daemonConfig.Storage.Allocations) > 0 {

		err := doOnce(ctx, func(ctx context.Context) error {
			if err := garageApplyLayout(ctx, logger, hostBootstrap, daemonConfig); err != nil {
				logger.Error(ctx, "applying garage layout", err)
				return err
			}

			return nil
		})

		if err != nil {
			return bootstrap.Bootstrap{}, fmt.Errorf("applying garage layout: %w", err)
		}
	}

	err = doOnce(ctx, func(ctx context.Context) error {
		if err := hostBootstrap.PutGarageBoostrapHost(ctx); err != nil {
			logger.Error(ctx, "updating host info in garage", err)
			return err
		}

		return nil
	})

	if err != nil {
		return bootstrap.Bootstrap{}, fmt.Errorf("updating host info in garage: %w", err)
	}

	ticker := time.NewTicker(3 * time.Minute)
	defer ticker.Stop()

	for {
		select {

		case <-ctx.Done():
			return bootstrap.Bootstrap{}, ctx.Err()

		case <-ticker.C:

			fmt.Fprintln(os.Stderr, "checking for changes to bootstrap")

			var (
				changed bool
				err     error
			)

			if hostBootstrap, changed, err = reloadBootstrap(ctx, logger, hostBootstrap); err != nil {
				return bootstrap.Bootstrap{}, fmt.Errorf("reloading bootstrap: %w", err)

			} else if changed {
				fmt.Fprintln(os.Stderr, "bootstrap info has changed, restarting all processes")
				return hostBootstrap, nil
			}
		}
	}
}

var subCmdDaemon = subCmd{
	name:  "daemon",
	descr: "Runs the isle daemon (Default if no sub-command given)",
	do: func(subCmdCtx subCmdCtx) error {

		flags := subCmdCtx.flagSet(false)

		daemonConfigPath := flags.StringP(
			"config-path", "c", "",
			"Optional path to a daemon.yml file to load configuration from.",
		)

		dumpConfig := flags.Bool(
			"dump-config", false,
			"Write the default configuration file to stdout and exit.",
		)

		bootstrapPath := flags.StringP(
			"bootstrap-path", "b", "",
			`Path to a bootstrap.yml file. This only needs to be provided the first time the daemon is started, after that it is ignored. If the isle binary has a bootstrap built into it then this argument is always optional.`,
		)

		logLevelStr := flags.StringP(
			"log-level", "l", "info",
			`Maximum log level which should be output. Values can be "debug", "info", "warn", "error", "fatal". Does not apply to sub-processes`,
		)

		if err := flags.Parse(subCmdCtx.args); err != nil {
			return fmt.Errorf("parsing flags: %w", err)
		}

		ctx := subCmdCtx.ctx

		if *dumpConfig {
			return daemon.CopyDefaultConfig(os.Stdout, envAppDirPath)
		}

		logLevel := mlog.LevelFromString(*logLevelStr)
		if logLevel == nil {
			return fmt.Errorf("couldn't parse log level %q", *logLevelStr)
		}

		logger := subCmdCtx.logger.WithMaxLevel(logLevel.Int())

		runtimeDirCleanup, err := setupAndLockRuntimeDir(ctx, logger)
		if err != nil {
			return fmt.Errorf("setting up runtime directory: %w", err)
		}
		defer runtimeDirCleanup()

		var (
			bootstrapDataDirPath = bootstrap.DataDirPath(envDataDirPath)
			bootstrapAppDirPath  = bootstrap.AppDirPath(envAppDirPath)

			hostBootstrapPath string
			hostBootstrap     bootstrap.Bootstrap
		)

		tryLoadBootstrap := func(path string) bool {

			if err != nil {
				return false

			} else if hostBootstrap, err = bootstrap.FromFile(path); errors.Is(err, fs.ErrNotExist) {
				fmt.Fprintf(os.Stderr, "bootstrap file not found at %q\n", path)
				err = nil
				return false

			} else if err != nil {
				err = fmt.Errorf("parsing bootstrap.yml at %q: %w", path, err)
				return false
			}

			logger.Info(
				mctx.Annotate(ctx, "bootstrapFilePath", path),
				"bootstrap file found",
			)

			hostBootstrapPath = path
			return true
		}

		switch {
		case tryLoadBootstrap(bootstrapDataDirPath):
		case *bootstrapPath != "" && tryLoadBootstrap(*bootstrapPath):
		case tryLoadBootstrap(bootstrapAppDirPath):
		case err != nil:
			return fmt.Errorf("attempting to load bootstrap.yml file: %w", err)
		default:
			return errors.New("No bootstrap.yml file could be found, and one is not provided with --bootstrap-path")
		}

		if hostBootstrapPath != bootstrapDataDirPath {

			// If the bootstrap file is not being stored in the data dir, copy
			// it there, so it can be loaded from there next time.
			if err := writeBootstrapToDataDir(hostBootstrap); err != nil {
				return fmt.Errorf("writing bootstrap.yml to data dir: %w", err)
			}
		}

		daemonConfig, err := daemon.LoadConfig(envAppDirPath, *daemonConfigPath)
		if err != nil {
			return fmt.Errorf("loading daemon config: %w", err)
		}

		// we update this Host's data using whatever configuration has been
		// provided by the daemon config. This way the daemon has the most
		// up-to-date possible bootstrap. This updated bootstrap will later get
		// updated in garage using bootstrap.PutGarageBoostrapHost, so other
		// hosts will see it as well.
		if hostBootstrap, daemonConfig, err = coalesceDaemonConfigAndBootstrap(hostBootstrap, daemonConfig); err != nil {
			return fmt.Errorf("merging daemon config into bootstrap data: %w", err)
		}

		for {

			hostBootstrap, err = runDaemonPmuxOnce(ctx, logger, hostBootstrap, daemonConfig)

			if errors.Is(err, context.Canceled) {
				return nil

			} else if err != nil {
				return fmt.Errorf("running pmux for daemon: %w", err)
			}
		}
	},
}