2021-04-20 21:31:37 +00:00
package entrypoint
import (
"bytes"
"context"
"errors"
"fmt"
"os"
"sync"
"time"
crypticnet "cryptic-net"
"cryptic-net/bootstrap"
2022-10-15 16:41:07 +00:00
"cryptic-net/garage"
2021-04-20 21:31:37 +00:00
"github.com/cryptic-io/pmux/pmuxlib"
)
// The daemon sub-command deals with starting an actual cryptic-net daemon
// process, which is required to be running for most other cryptic-net
// functionality. The sub-command does the following:
//
// * Creates and locks the runtime directory.
//
// * Creates the data directory and copies the appdir bootstrap file into there,
// if it's not already there.
//
// * Merges the user-provided daemon.yml file with the default, and writes the
// result to the runtime dir.
//
2022-10-16 13:38:15 +00:00
// * Merges daemon.yml configuration into the bootstrap configuration, and
// rewrites the bootstrap file.
//
2021-04-20 21:31:37 +00:00
// * Sets up environment variables that all other sub-processes then use, based
// on the runtime dir.
//
// * Dynamically creates the root pmux config and runs pmux.
//
// * (On exit) cleans up the runtime directory.
// creates a new bootstrap file using available information from the network. If
// the new bootstrap file is different than the existing one, the existing one
2022-10-19 14:20:26 +00:00
// is overwritten, env's bootstrap is reloaded, true is returned.
func reloadBootstrap ( env crypticnet . Env , s3Client garage . S3APIClient ) ( crypticnet . Env , bool , error ) {
2021-04-20 21:31:37 +00:00
2022-10-15 16:41:07 +00:00
newHosts , err := bootstrap . GetGarageBootstrapHosts ( env . Context , s3Client )
if err != nil {
2022-10-19 14:20:26 +00:00
return crypticnet . Env { } , false , fmt . Errorf ( "getting hosts from garage: %w" , err )
2022-10-15 16:41:07 +00:00
}
2021-04-20 21:31:37 +00:00
2022-10-15 16:41:07 +00:00
newHostsHash , err := bootstrap . HostsHash ( newHosts )
if err != nil {
2022-10-19 14:20:26 +00:00
return crypticnet . Env { } , false , fmt . Errorf ( "calculating hash of new hosts: %w" , err )
2021-04-20 21:31:37 +00:00
}
2022-10-15 16:41:07 +00:00
currHostsHash , err := bootstrap . HostsHash ( env . Bootstrap . Hosts )
2021-04-20 21:31:37 +00:00
if err != nil {
2022-10-19 14:20:26 +00:00
return crypticnet . Env { } , false , fmt . Errorf ( "calculating hash of current hosts: %w" , err )
2021-04-20 21:31:37 +00:00
}
2022-10-15 16:41:07 +00:00
if bytes . Equal ( newHostsHash , currHostsHash ) {
2022-10-19 14:20:26 +00:00
return crypticnet . Env { } , false , nil
2021-04-20 21:31:37 +00:00
}
2022-10-15 16:41:07 +00:00
buf := new ( bytes . Buffer )
if err := env . Bootstrap . WithHosts ( newHosts ) . WriteTo ( buf ) ; err != nil {
2022-10-19 14:20:26 +00:00
return crypticnet . Env { } , false , fmt . Errorf ( "writing new bootstrap file to buffer: %w" , err )
2022-10-15 16:41:07 +00:00
}
2022-10-19 14:20:26 +00:00
if env , err = copyBootstrapToDataDirAndReload ( env , buf ) ; err != nil {
return crypticnet . Env { } , false , fmt . Errorf ( "copying new bootstrap file to data dir: %w" , err )
2021-04-20 21:31:37 +00:00
}
2022-10-19 14:20:26 +00:00
return env , true , nil
2021-04-20 21:31:37 +00:00
}
2022-10-15 14:28:03 +00:00
// runs a single pmux process ofor daemon, returning only once the env.Context
2021-04-20 21:31:37 +00:00
// has been canceled or bootstrap info has been changed. This will always block
// until the spawned pmux has returned.
2022-10-19 14:20:26 +00:00
func runDaemonPmuxOnce ( env crypticnet . Env , s3Client garage . S3APIClient ) error {
2021-04-20 21:31:37 +00:00
2022-10-15 14:28:03 +00:00
thisHost := env . Bootstrap . ThisHost ( )
2022-10-15 16:41:07 +00:00
thisDaemon := env . ThisDaemon ( )
2021-04-20 21:31:37 +00:00
fmt . Fprintf ( os . Stderr , "host name is %q, ip is %q\n" , thisHost . Name , thisHost . Nebula . IP )
pmuxProcConfigs := [ ] pmuxlib . ProcessConfig {
2022-10-16 18:33:31 +00:00
nebulaEntrypointPmuxProcConfig ( ) ,
2021-04-20 21:31:37 +00:00
{
Name : "dnsmasq" ,
Cmd : "bash" ,
2022-10-16 18:33:31 +00:00
Args : waitForNebulaArgs ( env , "dnsmasq-entrypoint" ) ,
2021-04-20 21:31:37 +00:00
} ,
}
2022-10-16 18:33:31 +00:00
if len ( thisDaemon . Storage . Allocations ) > 0 {
garageChildrenPmuxProcConfigs , err := garageChildrenPmuxProcConfigs ( env )
if err != nil {
return fmt . Errorf ( "generating garage children configs: %w" , err )
2022-10-15 16:41:07 +00:00
}
2022-10-16 18:33:31 +00:00
pmuxProcConfigs = append ( pmuxProcConfigs , garageChildrenPmuxProcConfigs ... )
2022-10-15 16:41:07 +00:00
}
2022-10-16 18:33:31 +00:00
pmuxProcConfigs = append ( pmuxProcConfigs , pmuxlib . ProcessConfig {
Name : "update-global-bucket" ,
Cmd : "bash" ,
Args : waitForGarageArgs ( env , "update-global-bucket" ) ,
NoRestartOn : [ ] int { 0 } ,
} )
2021-04-20 21:31:37 +00:00
pmuxConfig := pmuxlib . Config { Processes : pmuxProcConfigs }
doneCh := env . Context . Done ( )
var wg sync . WaitGroup
defer wg . Wait ( )
ctx , cancel := context . WithCancel ( env . Context )
defer cancel ( )
wg . Add ( 1 )
go func ( ) {
defer wg . Done ( )
pmuxlib . Run ( ctx , pmuxConfig )
} ( )
2022-10-19 14:20:26 +00:00
if len ( thisDaemon . Storage . Allocations ) > 0 {
wg . Add ( 1 )
go func ( ) {
defer wg . Done ( )
if err := waitForGarage ( ctx , env ) ; err != nil {
fmt . Fprintf ( os . Stderr , "aborted waiting for garage instances to start: %v\n" , err )
return
}
err := doOnce ( ctx , func ( ctx context . Context ) error {
return garageApplyLayout ( ctx , env )
} )
if err != nil {
fmt . Fprintf ( os . Stderr , "aborted applying garage layout: %v\n" , err )
}
} ( )
}
2021-04-20 21:31:37 +00:00
ticker := time . NewTicker ( 3 * time . Minute )
defer ticker . Stop ( )
for {
select {
case <- doneCh :
return env . Context . Err ( )
case <- ticker . C :
fmt . Fprintln ( os . Stderr , "checking for changes to bootstrap" )
2022-10-19 14:20:26 +00:00
var (
changed bool
err error
)
if env , changed , err = reloadBootstrap ( env , s3Client ) ; err != nil {
2021-04-20 21:31:37 +00:00
return fmt . Errorf ( "reloading bootstrap: %w" , err )
} else if changed {
fmt . Fprintln ( os . Stderr , "bootstrap info has changed, restarting all processes" )
return nil
}
}
}
}
var subCmdDaemon = subCmd {
name : "daemon" ,
descr : "Runs the cryptic-net daemon (Default if no sub-command given)" ,
do : func ( subCmdCtx subCmdCtx ) error {
flags := subCmdCtx . flagSet ( false )
daemonYmlPath := flags . StringP (
"config-path" , "c" , "" ,
"Optional path to a daemon.yml file to load configuration from." ,
)
dumpConfig := flags . Bool (
"dump-config" , false ,
"Write the default configuration file to stdout and exit." ,
)
bootstrapPath := flags . StringP (
"bootstrap-path" , "b" , "" ,
` Path to a bootstrap.tgz file. This only needs to be provided the first time the daemon is started, after that it is ignored. If the cryptic-net binary has a bootstrap built into it then this argument is always optional. ` ,
)
if err := flags . Parse ( subCmdCtx . args ) ; err != nil {
return fmt . Errorf ( "parsing flags: %w" , err )
}
env := subCmdCtx . env
if * dumpConfig {
2022-10-16 18:33:31 +00:00
return writeBuiltinDaemonYml ( env , os . Stdout )
2021-04-20 21:31:37 +00:00
}
runtimeDirPath := env . RuntimeDirPath
fmt . Fprintf ( os . Stderr , "will use runtime directory %q for temporary state\n" , runtimeDirPath )
if err := os . MkdirAll ( runtimeDirPath , 0700 ) ; err != nil {
return fmt . Errorf ( "creating directory %q: %w" , runtimeDirPath , err )
} else if err := crypticnet . NewProcLock ( runtimeDirPath ) . WriteLock ( ) ; err != nil {
return err
}
// do not defer the cleaning of the runtime directory until the lock has
// been obtained, otherwise we might delete the directory out from under
// the feet of an already running daemon
defer func ( ) {
fmt . Fprintf ( os . Stderr , "cleaning up runtime directory %q\n" , runtimeDirPath )
if err := os . RemoveAll ( runtimeDirPath ) ; err != nil {
fmt . Fprintf ( os . Stderr , "error removing temporary directory %q: %v" , runtimeDirPath , err )
}
} ( )
// If the bootstrap file is not being stored in the data dir, move it
// there and reload the bootstrap info
if env . BootstrapPath != env . DataDirBootstrapPath ( ) {
path := env . BootstrapPath
// If there's no BootstrapPath then no bootstrap file could be
// found. In this case we require the user to provide one on the
// command-line.
if path == "" {
if * bootstrapPath == "" {
return errors . New ( "No bootstrap.tgz file could be found, and one is not provided with --bootstrap-path" )
}
path = * bootstrapPath
}
f , err := os . Open ( path )
if err != nil {
return fmt . Errorf ( "opening file %q: %w" , env . BootstrapPath , err )
}
2022-10-19 14:20:26 +00:00
env , err = copyBootstrapToDataDirAndReload ( env , f )
2021-04-20 21:31:37 +00:00
f . Close ( )
if err != nil {
return fmt . Errorf ( "copying bootstrap file from %q: %w" , path , err )
}
}
2022-10-16 18:33:31 +00:00
if err := writeMergedDaemonYml ( env , * daemonYmlPath ) ; err != nil {
return fmt . Errorf ( "merging and writing daemon.yml file: %w" , err )
2021-04-20 21:31:37 +00:00
}
2022-10-19 14:20:26 +00:00
var err error
2022-10-16 18:33:31 +00:00
// we update this Host's data using whatever configuration has been
// provided by daemon.yml. This way the daemon has the most
// up-to-date possible bootstrap. This updated bootstrap will later
// get updated in garage using update-global-bucket, so other hosts
// will see it as well.
2022-10-19 14:20:26 +00:00
if env , err = mergeDaemonIntoBootstrap ( env ) ; err != nil {
2022-10-16 18:33:31 +00:00
return fmt . Errorf ( "merging daemon.yml into bootstrap data: %w" , err )
2022-10-16 13:38:15 +00:00
}
2021-04-20 21:31:37 +00:00
for key , val := range env . ToMap ( ) {
if err := os . Setenv ( key , val ) ; err != nil {
return fmt . Errorf ( "failed to set %q to %q: %w" , key , val , err )
}
}
for {
2022-10-16 18:33:31 +00:00
// create s3Client anew on every loop, in case the topology has
// changed and we should be connecting to a different garage
// endpoint.
s3Client , err := env . Bootstrap . GlobalBucketS3APIClient ( )
if err != nil {
return fmt . Errorf ( "creating client for global bucket: %w" , err )
}
2022-10-15 16:41:07 +00:00
if err := runDaemonPmuxOnce ( env , s3Client ) ; errors . Is ( err , context . Canceled ) {
2021-04-20 21:31:37 +00:00
return nil
} else if err != nil {
return fmt . Errorf ( "running pmux for daemon: %w" , err )
}
}
} ,
}