325 lines
8.9 KiB
Go
325 lines
8.9 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"os"
|
|
"sync"
|
|
"time"
|
|
|
|
"isle/bootstrap"
|
|
"isle/daemon"
|
|
|
|
"code.betamike.com/micropelago/pmux/pmuxlib"
|
|
"github.com/mediocregopher/mediocre-go-lib/v2/mctx"
|
|
"github.com/mediocregopher/mediocre-go-lib/v2/mlog"
|
|
)
|
|
|
|
// The daemon sub-command deals with starting an actual isle daemon
|
|
// process, which is required to be running for most other Isle
|
|
// functionality. The sub-command does the following:
|
|
//
|
|
// * Creates and locks the runtime directory.
|
|
//
|
|
// * Creates the data directory and copies the appdir bootstrap file into there,
|
|
// if it's not already there.
|
|
//
|
|
// * Merges daemon configuration into the bootstrap configuration, and rewrites
|
|
// the bootstrap file.
|
|
//
|
|
// * Sets up environment variables that all other sub-processes then use, based
|
|
// on the runtime dir.
|
|
//
|
|
// * Dynamically creates the root pmux config and runs pmux.
|
|
//
|
|
// * (On exit) cleans up the runtime directory.
|
|
|
|
// creates a new bootstrap file using available information from the network. If
|
|
// the new bootstrap file is different than the existing one, the existing one
|
|
// is overwritten and true is returned.
|
|
func reloadBootstrap(
|
|
ctx context.Context,
|
|
logger *mlog.Logger,
|
|
hostBootstrap bootstrap.Bootstrap,
|
|
) (
|
|
bootstrap.Bootstrap, bool, error,
|
|
) {
|
|
|
|
thisHost := hostBootstrap.ThisHost()
|
|
|
|
newHosts, err := hostBootstrap.GetGarageBootstrapHosts(ctx, logger)
|
|
if err != nil {
|
|
return bootstrap.Bootstrap{}, false, fmt.Errorf("getting hosts from garage: %w", err)
|
|
}
|
|
|
|
// the daemon's view of this host's bootstrap info takes precedence over
|
|
// whatever is in garage
|
|
newHosts[thisHost.Name] = thisHost
|
|
|
|
newHostsHash, err := bootstrap.HostsHash(newHosts)
|
|
if err != nil {
|
|
return bootstrap.Bootstrap{}, false, fmt.Errorf("calculating hash of new hosts: %w", err)
|
|
}
|
|
|
|
currHostsHash, err := bootstrap.HostsHash(hostBootstrap.Hosts)
|
|
if err != nil {
|
|
return bootstrap.Bootstrap{}, false, fmt.Errorf("calculating hash of current hosts: %w", err)
|
|
}
|
|
|
|
if bytes.Equal(newHostsHash, currHostsHash) {
|
|
return hostBootstrap, false, nil
|
|
}
|
|
|
|
hostBootstrap.Hosts = newHosts
|
|
|
|
if err := writeBootstrapToDataDir(hostBootstrap); err != nil {
|
|
return bootstrap.Bootstrap{}, false, fmt.Errorf("writing new bootstrap to data dir: %w", err)
|
|
}
|
|
|
|
return hostBootstrap, true, nil
|
|
}
|
|
|
|
// runs a single pmux process of daemon, returning only once the env.Context has
|
|
// been canceled or bootstrap info has been changed. This will always block
|
|
// until the spawned pmux has returned, and returns a copy of hostBootstrap with
|
|
// updated boostrap info.
|
|
func runDaemonPmuxOnce(
|
|
ctx context.Context,
|
|
logger *mlog.Logger,
|
|
hostBootstrap bootstrap.Bootstrap,
|
|
daemonConfig daemon.Config,
|
|
) (
|
|
bootstrap.Bootstrap, error,
|
|
) {
|
|
|
|
nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(hostBootstrap, daemonConfig)
|
|
if err != nil {
|
|
return bootstrap.Bootstrap{}, fmt.Errorf("generating nebula config: %w", err)
|
|
}
|
|
|
|
dnsmasqPmuxProcConfig, err := dnsmasqPmuxProcConfig(hostBootstrap, daemonConfig)
|
|
if err != nil {
|
|
return bootstrap.Bootstrap{}, fmt.Errorf("generating dnsmasq config: %w", err)
|
|
}
|
|
|
|
garagePmuxProcConfigs, err := garagePmuxProcConfigs(hostBootstrap, daemonConfig)
|
|
if err != nil {
|
|
return bootstrap.Bootstrap{}, fmt.Errorf("generating garage children configs: %w", err)
|
|
}
|
|
|
|
pmuxConfig := pmuxlib.Config{
|
|
Processes: append(
|
|
[]pmuxlib.ProcessConfig{
|
|
nebulaPmuxProcConfig,
|
|
dnsmasqPmuxProcConfig,
|
|
},
|
|
garagePmuxProcConfigs...,
|
|
),
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
defer wg.Wait()
|
|
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
pmuxlib.Run(ctx, os.Stdout, os.Stderr, pmuxConfig)
|
|
}()
|
|
|
|
if err := waitForGarageAndNebula(ctx, logger, hostBootstrap, daemonConfig); err != nil {
|
|
return bootstrap.Bootstrap{}, fmt.Errorf("waiting for nebula/garage to start up: %w", err)
|
|
}
|
|
|
|
if len(daemonConfig.Storage.Allocations) > 0 {
|
|
|
|
err := doOnce(ctx, func(ctx context.Context) error {
|
|
if err := garageApplyLayout(ctx, logger, hostBootstrap, daemonConfig); err != nil {
|
|
logger.Error(ctx, "applying garage layout", err)
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
})
|
|
|
|
if err != nil {
|
|
return bootstrap.Bootstrap{}, fmt.Errorf("applying garage layout: %w", err)
|
|
}
|
|
}
|
|
|
|
err = doOnce(ctx, func(ctx context.Context) error {
|
|
if err := hostBootstrap.PutGarageBoostrapHost(ctx); err != nil {
|
|
logger.Error(ctx, "updating host info in garage", err)
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
})
|
|
|
|
if err != nil {
|
|
return bootstrap.Bootstrap{}, fmt.Errorf("updating host info in garage: %w", err)
|
|
}
|
|
|
|
ticker := time.NewTicker(3 * time.Minute)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
return bootstrap.Bootstrap{}, ctx.Err()
|
|
|
|
case <-ticker.C:
|
|
|
|
fmt.Fprintln(os.Stderr, "checking for changes to bootstrap")
|
|
|
|
var (
|
|
changed bool
|
|
err error
|
|
)
|
|
|
|
if hostBootstrap, changed, err = reloadBootstrap(ctx, logger, hostBootstrap); err != nil {
|
|
return bootstrap.Bootstrap{}, fmt.Errorf("reloading bootstrap: %w", err)
|
|
|
|
} else if changed {
|
|
fmt.Fprintln(os.Stderr, "bootstrap info has changed, restarting all processes")
|
|
return hostBootstrap, nil
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
var subCmdDaemon = subCmd{
|
|
name: "daemon",
|
|
descr: "Runs the isle daemon (Default if no sub-command given)",
|
|
do: func(subCmdCtx subCmdCtx) error {
|
|
|
|
flags := subCmdCtx.flagSet(false)
|
|
|
|
daemonConfigPath := flags.StringP(
|
|
"config-path", "c", "",
|
|
"Optional path to a daemon.yml file to load configuration from.",
|
|
)
|
|
|
|
dumpConfig := flags.Bool(
|
|
"dump-config", false,
|
|
"Write the default configuration file to stdout and exit.",
|
|
)
|
|
|
|
bootstrapPath := flags.StringP(
|
|
"bootstrap-path", "b", "",
|
|
`Path to a bootstrap.yml file. This only needs to be provided the first time the daemon is started, after that it is ignored. If the isle binary has a bootstrap built into it then this argument is always optional.`,
|
|
)
|
|
|
|
logLevelStr := flags.StringP(
|
|
"log-level", "l", "info",
|
|
`Maximum log level which should be output. Values can be "debug", "info", "warn", "error", "fatal". Does not apply to sub-processes`,
|
|
)
|
|
|
|
if err := flags.Parse(subCmdCtx.args); err != nil {
|
|
return fmt.Errorf("parsing flags: %w", err)
|
|
}
|
|
|
|
ctx := subCmdCtx.ctx
|
|
|
|
if *dumpConfig {
|
|
return daemon.CopyDefaultConfig(os.Stdout, envAppDirPath)
|
|
}
|
|
|
|
logLevel := mlog.LevelFromString(*logLevelStr)
|
|
if logLevel == nil {
|
|
return fmt.Errorf("couldn't parse log level %q", *logLevelStr)
|
|
}
|
|
|
|
logger := subCmdCtx.logger.WithMaxLevel(logLevel.Int())
|
|
|
|
runtimeDirCleanup, err := setupAndLockRuntimeDir(ctx, logger)
|
|
if err != nil {
|
|
return fmt.Errorf("setting up runtime directory: %w", err)
|
|
}
|
|
defer runtimeDirCleanup()
|
|
|
|
var (
|
|
bootstrapDataDirPath = bootstrap.DataDirPath(envDataDirPath)
|
|
bootstrapAppDirPath = bootstrap.AppDirPath(envAppDirPath)
|
|
|
|
hostBootstrapPath string
|
|
hostBootstrap bootstrap.Bootstrap
|
|
)
|
|
|
|
tryLoadBootstrap := func(path string) bool {
|
|
|
|
if err != nil {
|
|
return false
|
|
|
|
} else if hostBootstrap, err = bootstrap.FromFile(path); errors.Is(err, fs.ErrNotExist) {
|
|
fmt.Fprintf(os.Stderr, "bootstrap file not found at %q\n", path)
|
|
err = nil
|
|
return false
|
|
|
|
} else if err != nil {
|
|
err = fmt.Errorf("parsing bootstrap.yml at %q: %w", path, err)
|
|
return false
|
|
}
|
|
|
|
logger.Info(
|
|
mctx.Annotate(ctx, "bootstrapFilePath", path),
|
|
"bootstrap file found",
|
|
)
|
|
|
|
hostBootstrapPath = path
|
|
return true
|
|
}
|
|
|
|
switch {
|
|
case tryLoadBootstrap(bootstrapDataDirPath):
|
|
case *bootstrapPath != "" && tryLoadBootstrap(*bootstrapPath):
|
|
case tryLoadBootstrap(bootstrapAppDirPath):
|
|
case err != nil:
|
|
return fmt.Errorf("attempting to load bootstrap.yml file: %w", err)
|
|
default:
|
|
return errors.New("No bootstrap.yml file could be found, and one is not provided with --bootstrap-path")
|
|
}
|
|
|
|
if hostBootstrapPath != bootstrapDataDirPath {
|
|
|
|
// If the bootstrap file is not being stored in the data dir, copy
|
|
// it there, so it can be loaded from there next time.
|
|
if err := writeBootstrapToDataDir(hostBootstrap); err != nil {
|
|
return fmt.Errorf("writing bootstrap.yml to data dir: %w", err)
|
|
}
|
|
}
|
|
|
|
daemonConfig, err := daemon.LoadConfig(envAppDirPath, *daemonConfigPath)
|
|
if err != nil {
|
|
return fmt.Errorf("loading daemon config: %w", err)
|
|
}
|
|
|
|
// we update this Host's data using whatever configuration has been
|
|
// provided by the daemon config. This way the daemon has the most
|
|
// up-to-date possible bootstrap. This updated bootstrap will later get
|
|
// updated in garage using bootstrap.PutGarageBoostrapHost, so other
|
|
// hosts will see it as well.
|
|
if hostBootstrap, daemonConfig, err = coalesceDaemonConfigAndBootstrap(hostBootstrap, daemonConfig); err != nil {
|
|
return fmt.Errorf("merging daemon config into bootstrap data: %w", err)
|
|
}
|
|
|
|
for {
|
|
|
|
hostBootstrap, err = runDaemonPmuxOnce(ctx, logger, hostBootstrap, daemonConfig)
|
|
|
|
if errors.Is(err, context.Canceled) {
|
|
return nil
|
|
|
|
} else if err != nil {
|
|
return fmt.Errorf("running pmux for daemon: %w", err)
|
|
}
|
|
}
|
|
},
|
|
}
|