af7c8dde32
The `bootstrap/creator` package is gone, almost as quickly as it arrived. The `Bootstrap` type is now able to write its own tgz file, and the two places where bootstrap files are being created pull the data down to do so and create the `Bootstrap` structs directly. The structure of the bootstrap file itself has been changed, now there's just a single `hosts` directory which contains files which are yaml encodings of the `Host` type, rather than having it be split into `nebula` and `garage` directories. This makes creating bootstrap files a lot easier.
381 lines
9.7 KiB
Go
381 lines
9.7 KiB
Go
package entrypoint
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"sync"
|
|
"time"
|
|
|
|
crypticnet "cryptic-net"
|
|
"cryptic-net/bootstrap"
|
|
"cryptic-net/garage"
|
|
"cryptic-net/yamlutil"
|
|
|
|
"github.com/cryptic-io/pmux/pmuxlib"
|
|
"github.com/imdario/mergo"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
// The daemon sub-command deals with starting an actual cryptic-net daemon
|
|
// process, which is required to be running for most other cryptic-net
|
|
// functionality. The sub-command does the following:
|
|
//
|
|
// * Creates and locks the runtime directory.
|
|
//
|
|
// * Creates the data directory and copies the appdir bootstrap file into there,
|
|
// if it's not already there.
|
|
//
|
|
// * Merges the user-provided daemon.yml file with the default, and writes the
|
|
// result to the runtime dir.
|
|
//
|
|
// * Sets up environment variables that all other sub-processes then use, based
|
|
// on the runtime dir.
|
|
//
|
|
// * Dynamically creates the root pmux config and runs pmux.
|
|
//
|
|
// * (On exit) cleans up the runtime directory.
|
|
|
|
func writeDaemonYml(userDaemonYmlPath, builtinDaemonYmlPath, runtimeDirPath string) error {
|
|
|
|
var fullDaemonYml map[string]interface{}
|
|
|
|
if err := yamlutil.LoadYamlFile(&fullDaemonYml, builtinDaemonYmlPath); err != nil {
|
|
return fmt.Errorf("parsing builtin daemon.yml file: %w", err)
|
|
}
|
|
|
|
if userDaemonYmlPath != "" {
|
|
|
|
var daemonYml map[string]interface{}
|
|
if err := yamlutil.LoadYamlFile(&daemonYml, userDaemonYmlPath); err != nil {
|
|
return fmt.Errorf("parsing %q: %w", userDaemonYmlPath, err)
|
|
}
|
|
|
|
err := mergo.Merge(&fullDaemonYml, daemonYml, mergo.WithOverride)
|
|
if err != nil {
|
|
return fmt.Errorf("merging contents of file %q: %w", userDaemonYmlPath, err)
|
|
}
|
|
}
|
|
|
|
fullDaemonYmlB, err := yaml.Marshal(fullDaemonYml)
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("yaml marshaling daemon config: %w", err)
|
|
}
|
|
|
|
daemonYmlPath := filepath.Join(runtimeDirPath, "daemon.yml")
|
|
|
|
if err := ioutil.WriteFile(daemonYmlPath, fullDaemonYmlB, 0400); err != nil {
|
|
return fmt.Errorf("writing daemon.yml file to %q: %w", daemonYmlPath, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func writeBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
|
|
|
|
path := env.DataDirBootstrapPath()
|
|
dirPath := filepath.Dir(path)
|
|
|
|
if err := os.MkdirAll(dirPath, 0700); err != nil {
|
|
return fmt.Errorf("creating directory %q: %w", dirPath, err)
|
|
}
|
|
|
|
f, err := os.Create(path)
|
|
if err != nil {
|
|
return fmt.Errorf("creating file %q: %w", path, err)
|
|
}
|
|
|
|
_, err = io.Copy(f, r)
|
|
f.Close()
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("writing new bootstrap file to %q: %w", path, err)
|
|
}
|
|
|
|
if err := env.LoadBootstrap(path); err != nil {
|
|
return fmt.Errorf("loading bootstrap from %q: %w", path, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// creates a new bootstrap file using available information from the network. If
|
|
// the new bootstrap file is different than the existing one, the existing one
|
|
// is overwritten, ReloadBootstrap is called on env, true is returned.
|
|
func reloadBootstrap(env *crypticnet.Env, s3Client garage.S3APIClient) (bool, error) {
|
|
|
|
newHosts, err := bootstrap.GetGarageBootstrapHosts(env.Context, s3Client)
|
|
if err != nil {
|
|
return false, fmt.Errorf("getting hosts from garage: %w", err)
|
|
}
|
|
|
|
newHostsHash, err := bootstrap.HostsHash(newHosts)
|
|
if err != nil {
|
|
return false, fmt.Errorf("calculating hash of new hosts: %w", err)
|
|
}
|
|
|
|
currHostsHash, err := bootstrap.HostsHash(env.Bootstrap.Hosts)
|
|
if err != nil {
|
|
return false, fmt.Errorf("calculating hash of current hosts: %w", err)
|
|
}
|
|
|
|
if bytes.Equal(newHostsHash, currHostsHash) {
|
|
return false, nil
|
|
}
|
|
|
|
buf := new(bytes.Buffer)
|
|
if err := env.Bootstrap.WithHosts(newHosts).WriteTo(buf); err != nil {
|
|
return false, fmt.Errorf("writing new bootstrap file: %w", err)
|
|
}
|
|
|
|
if err := writeBootstrapToDataDir(env, buf); err != nil {
|
|
return false, fmt.Errorf("writing new bootstrap file: %w", err)
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
|
|
// runs a single pmux process ofor daemon, returning only once the env.Context
|
|
// has been canceled or bootstrap info has been changed. This will always block
|
|
// until the spawned pmux has returned.
|
|
func runDaemonPmuxOnce(env *crypticnet.Env, s3Client garage.S3APIClient) error {
|
|
|
|
thisHost := env.Bootstrap.ThisHost()
|
|
thisDaemon := env.ThisDaemon()
|
|
fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP)
|
|
|
|
pmuxProcConfigs := []pmuxlib.ProcessConfig{
|
|
{
|
|
Name: "nebula",
|
|
Cmd: "cryptic-net-main",
|
|
Args: []string{
|
|
"nebula-entrypoint",
|
|
},
|
|
},
|
|
{
|
|
Name: "dnsmasq",
|
|
Cmd: "bash",
|
|
Args: []string{
|
|
"wait-for-ip",
|
|
thisHost.Nebula.IP,
|
|
"dnsmasq-entrypoint",
|
|
},
|
|
},
|
|
}
|
|
|
|
{
|
|
var args []string
|
|
|
|
if allocs := thisDaemon.Storage.Allocations; len(allocs) > 0 {
|
|
for _, alloc := range allocs {
|
|
args = append(
|
|
args,
|
|
"wait-for",
|
|
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
|
|
"--",
|
|
)
|
|
}
|
|
} else {
|
|
args = []string{
|
|
"wait-for-ip",
|
|
thisHost.Nebula.IP,
|
|
}
|
|
}
|
|
|
|
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
|
|
Name: "update-global-bucket",
|
|
Cmd: "bash",
|
|
Args: append(args, "update-global-bucket"),
|
|
NoRestartOn: []int{0},
|
|
})
|
|
}
|
|
|
|
if len(thisDaemon.Storage.Allocations) > 0 {
|
|
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
|
|
Name: "garage",
|
|
Cmd: "bash",
|
|
Args: []string{
|
|
"wait-for-ip",
|
|
thisHost.Nebula.IP,
|
|
"cryptic-net-main", "garage-entrypoint",
|
|
},
|
|
|
|
// garage can take a while to clean up
|
|
SigKillWait: (1 * time.Minute) + (10 * time.Second),
|
|
})
|
|
}
|
|
|
|
pmuxConfig := pmuxlib.Config{Processes: pmuxProcConfigs}
|
|
|
|
doneCh := env.Context.Done()
|
|
|
|
var wg sync.WaitGroup
|
|
defer wg.Wait()
|
|
|
|
ctx, cancel := context.WithCancel(env.Context)
|
|
defer cancel()
|
|
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
pmuxlib.Run(ctx, pmuxConfig)
|
|
}()
|
|
|
|
ticker := time.NewTicker(3 * time.Minute)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
|
|
case <-doneCh:
|
|
return env.Context.Err()
|
|
|
|
case <-ticker.C:
|
|
|
|
fmt.Fprintln(os.Stderr, "checking for changes to bootstrap")
|
|
|
|
if changed, err := reloadBootstrap(env, s3Client); err != nil {
|
|
return fmt.Errorf("reloading bootstrap: %w", err)
|
|
|
|
} else if changed {
|
|
fmt.Fprintln(os.Stderr, "bootstrap info has changed, restarting all processes")
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
var subCmdDaemon = subCmd{
|
|
name: "daemon",
|
|
descr: "Runs the cryptic-net daemon (Default if no sub-command given)",
|
|
do: func(subCmdCtx subCmdCtx) error {
|
|
|
|
flags := subCmdCtx.flagSet(false)
|
|
|
|
daemonYmlPath := flags.StringP(
|
|
"config-path", "c", "",
|
|
"Optional path to a daemon.yml file to load configuration from.",
|
|
)
|
|
|
|
dumpConfig := flags.Bool(
|
|
"dump-config", false,
|
|
"Write the default configuration file to stdout and exit.",
|
|
)
|
|
|
|
bootstrapPath := flags.StringP(
|
|
"bootstrap-path", "b", "",
|
|
`Path to a bootstrap.tgz file. This only needs to be provided the first time the daemon is started, after that it is ignored. If the cryptic-net binary has a bootstrap built into it then this argument is always optional.`,
|
|
)
|
|
|
|
if err := flags.Parse(subCmdCtx.args); err != nil {
|
|
return fmt.Errorf("parsing flags: %w", err)
|
|
}
|
|
|
|
env := subCmdCtx.env
|
|
|
|
s3Client, err := env.GlobalBucketS3APIClient()
|
|
if err != nil {
|
|
return fmt.Errorf("creating client for global bucket: %w", err)
|
|
}
|
|
|
|
appDirPath := env.AppDirPath
|
|
|
|
builtinDaemonYmlPath := filepath.Join(appDirPath, "etc", "daemon.yml")
|
|
|
|
if *dumpConfig {
|
|
|
|
builtinDaemonYml, err := os.ReadFile(builtinDaemonYmlPath)
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("reading default daemon.yml at %q: %w", builtinDaemonYmlPath, err)
|
|
}
|
|
|
|
if _, err := os.Stdout.Write(builtinDaemonYml); err != nil {
|
|
return fmt.Errorf("writing default daemon.yml to stdout: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
runtimeDirPath := env.RuntimeDirPath
|
|
|
|
fmt.Fprintf(os.Stderr, "will use runtime directory %q for temporary state\n", runtimeDirPath)
|
|
|
|
if err := os.MkdirAll(runtimeDirPath, 0700); err != nil {
|
|
return fmt.Errorf("creating directory %q: %w", runtimeDirPath, err)
|
|
|
|
} else if err := crypticnet.NewProcLock(runtimeDirPath).WriteLock(); err != nil {
|
|
return err
|
|
}
|
|
|
|
// do not defer the cleaning of the runtime directory until the lock has
|
|
// been obtained, otherwise we might delete the directory out from under
|
|
// the feet of an already running daemon
|
|
defer func() {
|
|
fmt.Fprintf(os.Stderr, "cleaning up runtime directory %q\n", runtimeDirPath)
|
|
if err := os.RemoveAll(runtimeDirPath); err != nil {
|
|
fmt.Fprintf(os.Stderr, "error removing temporary directory %q: %v", runtimeDirPath, err)
|
|
}
|
|
}()
|
|
|
|
// If the bootstrap file is not being stored in the data dir, move it
|
|
// there and reload the bootstrap info
|
|
if env.BootstrapPath != env.DataDirBootstrapPath() {
|
|
|
|
path := env.BootstrapPath
|
|
|
|
// If there's no BootstrapPath then no bootstrap file could be
|
|
// found. In this case we require the user to provide one on the
|
|
// command-line.
|
|
if path == "" {
|
|
|
|
if *bootstrapPath == "" {
|
|
return errors.New("No bootstrap.tgz file could be found, and one is not provided with --bootstrap-path")
|
|
}
|
|
|
|
path = *bootstrapPath
|
|
}
|
|
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return fmt.Errorf("opening file %q: %w", env.BootstrapPath, err)
|
|
}
|
|
|
|
err = writeBootstrapToDataDir(env, f)
|
|
f.Close()
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("copying bootstrap file from %q: %w", path, err)
|
|
}
|
|
}
|
|
|
|
if err := writeDaemonYml(*daemonYmlPath, builtinDaemonYmlPath, runtimeDirPath); err != nil {
|
|
return fmt.Errorf("generating daemon.yml file: %w", err)
|
|
}
|
|
|
|
for key, val := range env.ToMap() {
|
|
if err := os.Setenv(key, val); err != nil {
|
|
return fmt.Errorf("failed to set %q to %q: %w", key, val, err)
|
|
}
|
|
}
|
|
|
|
for {
|
|
|
|
if err := runDaemonPmuxOnce(env, s3Client); errors.Is(err, context.Canceled) {
|
|
return nil
|
|
|
|
} else if err != nil {
|
|
return fmt.Errorf("running pmux for daemon: %w", err)
|
|
}
|
|
}
|
|
},
|
|
}
|