isle/go-workspace/src/cmd/entrypoint/daemon.go

package entrypoint

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"os"
	"sync"
	"time"

	crypticnet "cryptic-net"
	"cryptic-net/bootstrap"
	"cryptic-net/garage"

	"github.com/cryptic-io/pmux/pmuxlib"
)

// The daemon sub-command deals with starting an actual cryptic-net daemon
// process, which is required to be running for most other cryptic-net
// functionality. The sub-command does the following:
//
// * Creates and locks the runtime directory.
//
// * Creates the data directory and copies the appdir bootstrap file into there,
// if it's not already there.
//
// * Merges the user-provided daemon.yml file with the default, and writes the
// result to the runtime dir.
//
// * Merges daemon.yml configuration into the bootstrap configuration, and
// rewrites the bootstrap file.
//
// * Sets up environment variables that all other sub-processes then use, based
// on the runtime dir.
//
// * Dynamically creates the root pmux config and runs pmux.
//
// * (On exit) cleans up the runtime directory.

// creates a new bootstrap file using available information from the network. If
// the new bootstrap file is different than the existing one, the existing one
// is overwritten, env's bootstrap is reloaded, true is returned.
func reloadBootstrap(env crypticnet.Env, s3Client garage.S3APIClient) (crypticnet.Env, bool, error) {

	newHosts, err := bootstrap.GetGarageBootstrapHosts(env.Context, s3Client)
	if err != nil {
		return crypticnet.Env{}, false, fmt.Errorf("getting hosts from garage: %w", err)
	}

	newHostsHash, err := bootstrap.HostsHash(newHosts)
	if err != nil {
		return crypticnet.Env{}, false, fmt.Errorf("calculating hash of new hosts: %w", err)
	}

	currHostsHash, err := bootstrap.HostsHash(env.Bootstrap.Hosts)
	if err != nil {
		return crypticnet.Env{}, false, fmt.Errorf("calculating hash of current hosts: %w", err)
	}

	if bytes.Equal(newHostsHash, currHostsHash) {
		return crypticnet.Env{}, false, nil
	}

	buf := new(bytes.Buffer)
	if err := env.Bootstrap.WithHosts(newHosts).WriteTo(buf); err != nil {
		return crypticnet.Env{}, false, fmt.Errorf("writing new bootstrap file to buffer: %w", err)
	}

	if env, err = copyBootstrapToDataDirAndReload(env, buf); err != nil {
		return crypticnet.Env{}, false, fmt.Errorf("copying new bootstrap file to data dir: %w", err)
	}

	return env, true, nil
}

// runs a single pmux process ofor daemon, returning only once the env.Context
// has been canceled or bootstrap info has been changed. This will always block
// until the spawned pmux has returned, and returns a copy of Env with updated
// boostrap info.
func runDaemonPmuxOnce(env crypticnet.Env, s3Client garage.S3APIClient) (crypticnet.Env, error) {

	thisHost := env.Bootstrap.ThisHost()
	thisDaemon := env.ThisDaemon()
	fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP)

	pmuxProcConfigs := []pmuxlib.ProcessConfig{
		nebulaEntrypointPmuxProcConfig(),
		{
			Name: "dnsmasq",
			Cmd:  "bash",
			Args: waitForNebulaArgs(env, "dnsmasq-entrypoint"),
		},
	}

	if len(thisDaemon.Storage.Allocations) > 0 {

		garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env)
		if err != nil {
			return crypticnet.Env{}, fmt.Errorf("generating garage children configs: %w", err)
		}

		pmuxProcConfigs = append(pmuxProcConfigs, garageChildrenPmuxProcConfigs...)
	}

	pmuxConfig := pmuxlib.Config{Processes: pmuxProcConfigs}

	doneCh := env.Context.Done()

	var wg sync.WaitGroup
	defer wg.Wait()

	ctx, cancel := context.WithCancel(env.Context)
	defer cancel()

	wg.Add(1)
	go func() {
		defer wg.Done()
		pmuxlib.Run(ctx, pmuxConfig)
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()

		// TODO wait for garage or nebula, depending on if allocs are present

		client := env.Bootstrap.GlobalBucketS3APIClient()
		thisHost := env.Bootstrap.ThisHost()

		err := doOnce(ctx, func(ctx context.Context) error {
			fmt.Fprintln(os.Stderr, "updating host info in garage")
			return bootstrap.PutGarageBoostrapHost(ctx, client, thisHost)
		})

		if err != nil {
			fmt.Fprintf(os.Stderr, "aborted updating host info in garage: %v\n", err)
		}
	}()

	if len(thisDaemon.Storage.Allocations) > 0 {
		wg.Add(1)
		go func() {
			defer wg.Done()

			if err := waitForGarage(ctx, env); err != nil {
				fmt.Fprintf(os.Stderr, "aborted waiting for garage instances to start: %v\n", err)
				return
			}

			err := doOnce(ctx, func(ctx context.Context) error {
				fmt.Fprintln(os.Stderr, "applying garage layout")
				return garageApplyLayout(ctx, env)
			})

			if err != nil {
				fmt.Fprintf(os.Stderr, "aborted applying garage layout: %v\n", err)
			}
		}()
	}

	ticker := time.NewTicker(3 * time.Minute)
	defer ticker.Stop()

	for {
		select {

		case <-doneCh:
			return crypticnet.Env{}, env.Context.Err()

		case <-ticker.C:

			fmt.Fprintln(os.Stderr, "checking for changes to bootstrap")

			var (
				changed bool
				err     error
			)

			if env, changed, err = reloadBootstrap(env, s3Client); err != nil {
				return crypticnet.Env{}, fmt.Errorf("reloading bootstrap: %w", err)

			} else if changed {
				fmt.Fprintln(os.Stderr, "bootstrap info has changed, restarting all processes")
				return env, nil
			}
		}
	}
}

var subCmdDaemon = subCmd{
	name:  "daemon",
	descr: "Runs the cryptic-net daemon (Default if no sub-command given)",
	do: func(subCmdCtx subCmdCtx) error {

		flags := subCmdCtx.flagSet(false)

		daemonYmlPath := flags.StringP(
			"config-path", "c", "",
			"Optional path to a daemon.yml file to load configuration from.",
		)

		dumpConfig := flags.Bool(
			"dump-config", false,
			"Write the default configuration file to stdout and exit.",
		)

		bootstrapPath := flags.StringP(
			"bootstrap-path", "b", "",
			`Path to a bootstrap.tgz file. This only needs to be provided the first time the daemon is started, after that it is ignored. If the cryptic-net binary has a bootstrap built into it then this argument is always optional.`,
		)

		if err := flags.Parse(subCmdCtx.args); err != nil {
			return fmt.Errorf("parsing flags: %w", err)
		}

		env := subCmdCtx.env

		if *dumpConfig {
			return writeBuiltinDaemonYml(env, os.Stdout)
		}

		runtimeDirPath := env.RuntimeDirPath

		fmt.Fprintf(os.Stderr, "will use runtime directory %q for temporary state\n", runtimeDirPath)

		if err := os.MkdirAll(runtimeDirPath, 0700); err != nil {
			return fmt.Errorf("creating directory %q: %w", runtimeDirPath, err)

		} else if err := crypticnet.NewProcLock(runtimeDirPath).WriteLock(); err != nil {
			return err
		}

		// do not defer the cleaning of the runtime directory until the lock has
		// been obtained, otherwise we might delete the directory out from under
		// the feet of an already running daemon
		defer func() {
			fmt.Fprintf(os.Stderr, "cleaning up runtime directory %q\n", runtimeDirPath)
			if err := os.RemoveAll(runtimeDirPath); err != nil {
				fmt.Fprintf(os.Stderr, "error removing temporary directory %q: %v", runtimeDirPath, err)
			}
		}()

		// If the bootstrap file is not being stored in the data dir, move it
		// there and reload the bootstrap info
		if env.BootstrapPath != env.DataDirBootstrapPath() {

			path := env.BootstrapPath

			// If there's no BootstrapPath then no bootstrap file could be
			// found. In this case we require the user to provide one on the
			// command-line.
			if path == "" {

				if *bootstrapPath == "" {
					return errors.New("No bootstrap.tgz file could be found, and one is not provided with --bootstrap-path")
				}

				path = *bootstrapPath
			}

			f, err := os.Open(path)
			if err != nil {
				return fmt.Errorf("opening file %q: %w", env.BootstrapPath, err)
			}

			env, err = copyBootstrapToDataDirAndReload(env, f)
			f.Close()

			if err != nil {
				return fmt.Errorf("copying bootstrap file from %q: %w", path, err)
			}
		}

		if err := writeMergedDaemonYml(env, *daemonYmlPath); err != nil {
			return fmt.Errorf("merging and writing daemon.yml file: %w", err)
		}

		var err error

		// we update this Host's data using whatever configuration has been
		// provided by daemon.yml. This way the daemon has the most
		// up-to-date possible bootstrap. This updated bootstrap will later
		// get updated in garage using update-global-bucket, so other hosts
		// will see it as well.
		if env, err = mergeDaemonIntoBootstrap(env); err != nil {
			return fmt.Errorf("merging daemon.yml into bootstrap data: %w", err)
		}

		for key, val := range env.ToMap() {
			if err := os.Setenv(key, val); err != nil {
				return fmt.Errorf("failed to set %q to %q: %w", key, val, err)
			}
		}

		for {

			// create s3Client anew on every loop, in case the topology has
			// changed and we should be connecting to a different garage
			// endpoint.
			s3Client := env.Bootstrap.GlobalBucketS3APIClient()

			if env, err = runDaemonPmuxOnce(env, s3Client); errors.Is(err, context.Canceled) {
				return nil

			} else if err != nil {
				return fmt.Errorf("running pmux for daemon: %w", err)
			}
		}
	},
}