package main import ( "bytes" "context" "errors" "fmt" "io/fs" "os" "sync" "time" "isle/bootstrap" "isle/daemon" "code.betamike.com/micropelago/pmux/pmuxlib" "github.com/mediocregopher/mediocre-go-lib/v2/mctx" "github.com/mediocregopher/mediocre-go-lib/v2/mlog" ) // The daemon sub-command deals with starting an actual isle daemon // process, which is required to be running for most other Isle // functionality. The sub-command does the following: // // * Creates and locks the runtime directory. // // * Creates the data directory and copies the appdir bootstrap file into there, // if it's not already there. // // * Merges daemon configuration into the bootstrap configuration, and rewrites // the bootstrap file. // // * Sets up environment variables that all other sub-processes then use, based // on the runtime dir. // // * Dynamically creates the root pmux config and runs pmux. // // * (On exit) cleans up the runtime directory. // creates a new bootstrap file using available information from the network. If // the new bootstrap file is different than the existing one, the existing one // is overwritten and true is returned. func reloadBootstrap( ctx context.Context, logger *mlog.Logger, hostBootstrap bootstrap.Bootstrap, ) ( bootstrap.Bootstrap, bool, error, ) { thisHost := hostBootstrap.ThisHost() newHosts, err := hostBootstrap.GetGarageBootstrapHosts(ctx, logger) if err != nil { return bootstrap.Bootstrap{}, false, fmt.Errorf("getting hosts from garage: %w", err) } // the daemon's view of this host's bootstrap info takes precedence over // whatever is in garage newHosts[thisHost.Name] = thisHost newHostsHash, err := bootstrap.HostsHash(newHosts) if err != nil { return bootstrap.Bootstrap{}, false, fmt.Errorf("calculating hash of new hosts: %w", err) } currHostsHash, err := bootstrap.HostsHash(hostBootstrap.Hosts) if err != nil { return bootstrap.Bootstrap{}, false, fmt.Errorf("calculating hash of current hosts: %w", err) } if bytes.Equal(newHostsHash, currHostsHash) { return hostBootstrap, false, nil } hostBootstrap.Hosts = newHosts if err := writeBootstrapToDataDir(hostBootstrap); err != nil { return bootstrap.Bootstrap{}, false, fmt.Errorf("writing new bootstrap to data dir: %w", err) } return hostBootstrap, true, nil } // runs a single pmux process of daemon, returning only once the env.Context has // been canceled or bootstrap info has been changed. This will always block // until the spawned pmux has returned, and returns a copy of hostBootstrap with // updated boostrap info. func runDaemonPmuxOnce( ctx context.Context, logger *mlog.Logger, hostBootstrap bootstrap.Bootstrap, daemonConfig daemon.Config, ) ( bootstrap.Bootstrap, error, ) { nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(hostBootstrap, daemonConfig) if err != nil { return bootstrap.Bootstrap{}, fmt.Errorf("generating nebula config: %w", err) } dnsmasqPmuxProcConfig, err := dnsmasqPmuxProcConfig(hostBootstrap, daemonConfig) if err != nil { return bootstrap.Bootstrap{}, fmt.Errorf("generating dnsmasq config: %w", err) } garagePmuxProcConfigs, err := garagePmuxProcConfigs(hostBootstrap, daemonConfig) if err != nil { return bootstrap.Bootstrap{}, fmt.Errorf("generating garage children configs: %w", err) } pmuxConfig := pmuxlib.Config{ Processes: append( []pmuxlib.ProcessConfig{ nebulaPmuxProcConfig, dnsmasqPmuxProcConfig, }, garagePmuxProcConfigs..., ), } var wg sync.WaitGroup defer wg.Wait() ctx, cancel := context.WithCancel(ctx) defer cancel() wg.Add(1) go func() { defer wg.Done() pmuxlib.Run(ctx, os.Stdout, os.Stderr, pmuxConfig) }() if err := waitForGarageAndNebula(ctx, logger, hostBootstrap, daemonConfig); err != nil { return bootstrap.Bootstrap{}, fmt.Errorf("waiting for nebula/garage to start up: %w", err) } if len(daemonConfig.Storage.Allocations) > 0 { err := doOnce(ctx, func(ctx context.Context) error { if err := garageApplyLayout(ctx, logger, hostBootstrap, daemonConfig); err != nil { logger.Error(ctx, "applying garage layout", err) return err } return nil }) if err != nil { return bootstrap.Bootstrap{}, fmt.Errorf("applying garage layout: %w", err) } } err = doOnce(ctx, func(ctx context.Context) error { if err := hostBootstrap.PutGarageBoostrapHost(ctx); err != nil { logger.Error(ctx, "updating host info in garage", err) return err } return nil }) if err != nil { return bootstrap.Bootstrap{}, fmt.Errorf("updating host info in garage: %w", err) } ticker := time.NewTicker(3 * time.Minute) defer ticker.Stop() for { select { case <-ctx.Done(): return bootstrap.Bootstrap{}, ctx.Err() case <-ticker.C: fmt.Fprintln(os.Stderr, "checking for changes to bootstrap") var ( changed bool err error ) if hostBootstrap, changed, err = reloadBootstrap(ctx, logger, hostBootstrap); err != nil { return bootstrap.Bootstrap{}, fmt.Errorf("reloading bootstrap: %w", err) } else if changed { fmt.Fprintln(os.Stderr, "bootstrap info has changed, restarting all processes") return hostBootstrap, nil } } } } var subCmdDaemon = subCmd{ name: "daemon", descr: "Runs the isle daemon (Default if no sub-command given)", do: func(subCmdCtx subCmdCtx) error { flags := subCmdCtx.flagSet(false) daemonConfigPath := flags.StringP( "config-path", "c", "", "Optional path to a daemon.yml file to load configuration from.", ) dumpConfig := flags.Bool( "dump-config", false, "Write the default configuration file to stdout and exit.", ) bootstrapPath := flags.StringP( "bootstrap-path", "b", "", `Path to a bootstrap.json file. This only needs to be provided the first time the daemon is started, after that it is ignored. If the isle binary has a bootstrap built into it then this argument is always optional.`, ) logLevelStr := flags.StringP( "log-level", "l", "info", `Maximum log level which should be output. Values can be "debug", "info", "warn", "error", "fatal". Does not apply to sub-processes`, ) if err := flags.Parse(subCmdCtx.args); err != nil { return fmt.Errorf("parsing flags: %w", err) } ctx := subCmdCtx.ctx if *dumpConfig { return daemon.CopyDefaultConfig(os.Stdout, envAppDirPath) } logLevel := mlog.LevelFromString(*logLevelStr) if logLevel == nil { return fmt.Errorf("couldn't parse log level %q", *logLevelStr) } logger := subCmdCtx.logger.WithMaxLevel(logLevel.Int()) runtimeDirCleanup, err := setupAndLockRuntimeDir(ctx, logger) if err != nil { return fmt.Errorf("setting up runtime directory: %w", err) } defer runtimeDirCleanup() var ( bootstrapDataDirPath = bootstrap.DataDirPath(envDataDirPath) bootstrapAppDirPath = bootstrap.AppDirPath(envAppDirPath) hostBootstrapPath string hostBootstrap bootstrap.Bootstrap ) tryLoadBootstrap := func(path string) bool { if err != nil { return false } else if hostBootstrap, err = bootstrap.FromFile(path); errors.Is(err, fs.ErrNotExist) { fmt.Fprintf(os.Stderr, "bootstrap file not found at %q\n", path) err = nil return false } else if err != nil { err = fmt.Errorf("parsing bootstrap.json at %q: %w", path, err) return false } logger.Info( mctx.Annotate(ctx, "bootstrapFilePath", path), "bootstrap file found", ) hostBootstrapPath = path return true } switch { case tryLoadBootstrap(bootstrapDataDirPath): case *bootstrapPath != "" && tryLoadBootstrap(*bootstrapPath): case tryLoadBootstrap(bootstrapAppDirPath): case err != nil: return fmt.Errorf("attempting to load bootstrap.json file: %w", err) default: return errors.New("No bootstrap.json file could be found, and one is not provided with --bootstrap-path") } if hostBootstrapPath != bootstrapDataDirPath { // If the bootstrap file is not being stored in the data dir, copy // it there, so it can be loaded from there next time. if err := writeBootstrapToDataDir(hostBootstrap); err != nil { return fmt.Errorf("writing bootstrap.json to data dir: %w", err) } } daemonConfig, err := daemon.LoadConfig(envAppDirPath, *daemonConfigPath) if err != nil { return fmt.Errorf("loading daemon config: %w", err) } // we update this Host's data using whatever configuration has been // provided by the daemon config. This way the daemon has the most // up-to-date possible bootstrap. This updated bootstrap will later get // updated in garage using bootstrap.PutGarageBoostrapHost, so other // hosts will see it as well. if hostBootstrap, daemonConfig, err = coalesceDaemonConfigAndBootstrap(hostBootstrap, daemonConfig); err != nil { return fmt.Errorf("merging daemon config into bootstrap data: %w", err) } for { hostBootstrap, err = runDaemonPmuxOnce(ctx, logger, hostBootstrap, daemonConfig) if errors.Is(err, context.Canceled) { return nil } else if err != nil { return fmt.Errorf("running pmux for daemon: %w", err) } } }, }