diff --git a/AppDir/AppRun b/AppDir/AppRun index 6ef0c8e..6e3cecf 100755 --- a/AppDir/AppRun +++ b/AppDir/AppRun @@ -1,4 +1,4 @@ #!/bin/sh export PATH=$APPDIR/bin -exec cryptic-net-main entrypoint "$@" +exec entrypoint "$@" diff --git a/default.nix b/default.nix index d42f271..c8fcbff 100644 --- a/default.nix +++ b/default.nix @@ -74,7 +74,7 @@ dnsmasq garage waitFor - goWorkspace.crypticNetMain + goWorkspace.entrypoint ] ++ (if bootstrap != null then [ rootedBootstrap ] else []); }; diff --git a/go-workspace/default.nix b/go-workspace/default.nix index d3a3cb3..41542ab 100644 --- a/go-workspace/default.nix +++ b/go-workspace/default.nix @@ -14,5 +14,5 @@ }; in { - crypticNetMain = build "cmd/cryptic-net-main"; + entrypoint = build "cmd/entrypoint"; } diff --git a/go-workspace/src/cmd/cryptic-net-main/main.go b/go-workspace/src/cmd/cryptic-net-main/main.go deleted file mode 100644 index 4efe623..0000000 --- a/go-workspace/src/cmd/cryptic-net-main/main.go +++ /dev/null @@ -1,73 +0,0 @@ -package main - -// -// This binary acts as a wrapper around other programs which would otherwise -// form their own binaries. We do this for two reasons: -// -// * Nix makes it difficult to determine which individuals binaries need to be -// rebuilt upon changes, so it rebuilds all of them no matter what changed. This -// makes development slow. By wrapping everything in a sinble binary we only -// ever have to build that binary. -// -// * If we have N binaries total, then we have N copies of the go runtime in our -// final AppImage. By bundling the binaries into a single one we can reduce the -// number go runtime copies to 1. -// - -import ( - "cryptic-net/cmd/entrypoint" - nebula_entrypoint "cryptic-net/cmd/nebula-entrypoint" - "fmt" - "os" -) - -type mainFn struct { - name string - fn func() -} - -var mainFns = []mainFn{ - {"entrypoint", entrypoint.Main}, - {"nebula-entrypoint", nebula_entrypoint.Main}, -} - -var mainFnsMap = func() map[string]mainFn { - - m := map[string]mainFn{} - - for _, mainFn := range mainFns { - m[mainFn.name] = mainFn - } - - return m -}() - -func usage() { - fmt.Fprintf(os.Stderr, "USAGE: %s \n\n", os.Args[0]) - fmt.Fprintf(os.Stderr, "Commands:\n\n") - - for _, mainFn := range mainFns { - fmt.Fprintf(os.Stderr, "%s\n", mainFn.name) - } - - os.Stderr.Sync() - os.Exit(1) -} - -func main() { - - if len(os.Args) < 2 { - usage() - } - - mainFn, ok := mainFnsMap[os.Args[1]] - if !ok { - usage() - } - - // remove os.Args[1] from the arg list, so that other commands which consume - // args don't get confused - os.Args = append(os.Args[:1], os.Args[2:]...) - - mainFn.fn() -} diff --git a/go-workspace/src/cmd/entrypoint/admin.go b/go-workspace/src/cmd/entrypoint/admin.go index 34073f5..29048fe 100644 --- a/go-workspace/src/cmd/entrypoint/admin.go +++ b/go-workspace/src/cmd/entrypoint/admin.go @@ -1,4 +1,4 @@ -package entrypoint +package main import ( "context" @@ -173,17 +173,22 @@ var subCmdAdminCreateNetwork = subCmd{ } } - garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env) + nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(env) if err != nil { - return fmt.Errorf("generating garage children configs: %w", err) + return fmt.Errorf("generating nebula config: %w", err) + } + + garagePmuxProcConfigs, err := garagePmuxProcConfigs(env) + if err != nil { + return fmt.Errorf("generating garage configs: %w", err) } pmuxConfig := pmuxlib.Config{ Processes: append( []pmuxlib.ProcessConfig{ - nebulaEntrypointPmuxProcConfig(), + nebulaPmuxProcConfig, }, - garageChildrenPmuxProcConfigs..., + garagePmuxProcConfigs..., ), } @@ -203,7 +208,7 @@ var subCmdAdminCreateNetwork = subCmd{ }() fmt.Fprintln(os.Stderr, "waiting for garage instances to come online") - if err := waitForGarage(ctx, env); err != nil { + if err := waitForGarageAndNebula(ctx, env); err != nil { return fmt.Errorf("waiting for garage to start up: %w", err) } diff --git a/go-workspace/src/cmd/entrypoint/daemon.go b/go-workspace/src/cmd/entrypoint/daemon.go index daf6c33..bad5699 100644 --- a/go-workspace/src/cmd/entrypoint/daemon.go +++ b/go-workspace/src/cmd/entrypoint/daemon.go @@ -1,4 +1,4 @@ -package entrypoint +package main import ( "bytes" @@ -74,33 +74,46 @@ func reloadBootstrap(env crypticnet.Env, s3Client garage.S3APIClient) (crypticne return env, true, nil } -// runs a single pmux process ofor daemon, returning only once the env.Context -// has been canceled or bootstrap info has been changed. This will always block +// runs a single pmux process of daemon, returning only once the env.Context has +// been canceled or bootstrap info has been changed. This will always block // until the spawned pmux has returned, and returns a copy of Env with updated // boostrap info. -func runDaemonPmuxOnce(env crypticnet.Env, s3Client garage.S3APIClient) (crypticnet.Env, error) { +func runDaemonPmuxOnce(env crypticnet.Env) (crypticnet.Env, error) { thisHost := env.Bootstrap.ThisHost() thisDaemon := env.ThisDaemon() fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP) + // create s3Client anew on every loop, in case the topology has + // changed and we should be connecting to a different garage + // endpoint. + s3Client := env.Bootstrap.GlobalBucketS3APIClient() + + nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(env) + if err != nil { + return crypticnet.Env{}, fmt.Errorf("generating nebula config: %w", err) + } + pmuxProcConfigs := []pmuxlib.ProcessConfig{ - nebulaEntrypointPmuxProcConfig(), + nebulaPmuxProcConfig, { Name: "dnsmasq", Cmd: "bash", - Args: waitForNebulaArgs(env, "dnsmasq-entrypoint"), + Args: []string{"dnsmasq-entrypoint"}, + StartAfterFunc: func(ctx context.Context) error { + return waitForNebula(ctx, env) + }, }, } if len(thisDaemon.Storage.Allocations) > 0 { - garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env) + garagePmuxProcConfigs, err := garagePmuxProcConfigs(env) if err != nil { return crypticnet.Env{}, fmt.Errorf("generating garage children configs: %w", err) } - pmuxProcConfigs = append(pmuxProcConfigs, garageChildrenPmuxProcConfigs...) + pmuxProcConfigs = append(pmuxProcConfigs, garagePmuxProcConfigs...) } pmuxConfig := pmuxlib.Config{Processes: pmuxProcConfigs} @@ -123,14 +136,16 @@ func runDaemonPmuxOnce(env crypticnet.Env, s3Client garage.S3APIClient) (cryptic go func() { defer wg.Done() - // TODO wait for garage or nebula, depending on if allocs are present + if err := waitForGarageAndNebula(ctx, env); err != nil { + fmt.Fprintf(os.Stderr, "aborted waiting for garage instances to be accessible: %v\n", err) + return + } - client := env.Bootstrap.GlobalBucketS3APIClient() thisHost := env.Bootstrap.ThisHost() err := doOnce(ctx, func(ctx context.Context) error { fmt.Fprintln(os.Stderr, "updating host info in garage") - return bootstrap.PutGarageBoostrapHost(ctx, client, thisHost) + return bootstrap.PutGarageBoostrapHost(ctx, s3Client, thisHost) }) if err != nil { @@ -143,8 +158,8 @@ func runDaemonPmuxOnce(env crypticnet.Env, s3Client garage.S3APIClient) (cryptic go func() { defer wg.Done() - if err := waitForGarage(ctx, env); err != nil { - fmt.Fprintf(os.Stderr, "aborted waiting for garage instances to start: %v\n", err) + if err := waitForGarageAndNebula(ctx, env); err != nil { + fmt.Fprintf(os.Stderr, "aborted waiting for garage instances to be accessible: %v\n", err) return } @@ -287,6 +302,9 @@ var subCmdDaemon = subCmd{ return fmt.Errorf("merging daemon.yml into bootstrap data: %w", err) } + // TODO once dnsmasq entrypoint is written in go and the config + // generation is inlined into this process then this Setenv won't be + // necessary. for key, val := range env.ToMap() { if err := os.Setenv(key, val); err != nil { return fmt.Errorf("failed to set %q to %q: %w", key, val, err) @@ -295,12 +313,7 @@ var subCmdDaemon = subCmd{ for { - // create s3Client anew on every loop, in case the topology has - // changed and we should be connecting to a different garage - // endpoint. - s3Client := env.Bootstrap.GlobalBucketS3APIClient() - - if env, err = runDaemonPmuxOnce(env, s3Client); errors.Is(err, context.Canceled) { + if env, err = runDaemonPmuxOnce(env); errors.Is(err, context.Canceled) { return nil } else if err != nil { diff --git a/go-workspace/src/cmd/entrypoint/daemon_util.go b/go-workspace/src/cmd/entrypoint/daemon_util.go index dae1473..d19fe45 100644 --- a/go-workspace/src/cmd/entrypoint/daemon_util.go +++ b/go-workspace/src/cmd/entrypoint/daemon_util.go @@ -1,4 +1,4 @@ -package entrypoint +package main import ( "bytes" @@ -9,8 +9,7 @@ import ( "io" "os" "path/filepath" - - "code.betamike.com/cryptic-io/pmux/pmuxlib" + "time" ) func copyBootstrapToDataDirAndReload(env crypticnet.Env, r io.Reader) (crypticnet.Env, error) { @@ -74,20 +73,7 @@ func doOnce(ctx context.Context, fn func(context.Context) error) error { } else if ctxErr := ctx.Err(); ctxErr != nil { return ctxErr } - } -} - -func waitForNebulaArgs(env crypticnet.Env, args ...string) []string { - thisHost := env.Bootstrap.ThisHost() - return append([]string{"wait-for-ip", thisHost.Nebula.IP}, args...) -} - -func nebulaEntrypointPmuxProcConfig() pmuxlib.ProcessConfig { - return pmuxlib.ProcessConfig{ - Name: "nebula", - Cmd: "cryptic-net-main", - Args: []string{ - "nebula-entrypoint", - }, + + time.Sleep(250 * time.Millisecond) } } diff --git a/go-workspace/src/cmd/entrypoint/daemon_yml.go b/go-workspace/src/cmd/entrypoint/daemon_yml.go index cc4f5ed..f12efe4 100644 --- a/go-workspace/src/cmd/entrypoint/daemon_yml.go +++ b/go-workspace/src/cmd/entrypoint/daemon_yml.go @@ -1,4 +1,4 @@ -package entrypoint +package main import ( crypticnet "cryptic-net" diff --git a/go-workspace/src/cmd/entrypoint/garage.go b/go-workspace/src/cmd/entrypoint/garage.go index e516810..a684cfb 100644 --- a/go-workspace/src/cmd/entrypoint/garage.go +++ b/go-workspace/src/cmd/entrypoint/garage.go @@ -1,4 +1,4 @@ -package entrypoint +package main import ( "fmt" diff --git a/go-workspace/src/cmd/entrypoint/garage_util.go b/go-workspace/src/cmd/entrypoint/garage_util.go index 8811f24..e4ce8a5 100644 --- a/go-workspace/src/cmd/entrypoint/garage_util.go +++ b/go-workspace/src/cmd/entrypoint/garage_util.go @@ -1,4 +1,4 @@ -package entrypoint +package main import ( "context" @@ -9,14 +9,21 @@ import ( "os" "path/filepath" "strconv" - "time" "code.betamike.com/cryptic-io/pmux/pmuxlib" ) -func waitForGarage(ctx context.Context, env crypticnet.Env) error { +func waitForGarageAndNebula(ctx context.Context, env crypticnet.Env) error { - for _, alloc := range env.ThisDaemon().Storage.Allocations { + allocs := env.ThisDaemon().Storage.Allocations + + // if this host doesn't have any allocations specified then fall back to + // waiting for nebula + if len(allocs) == 0 { + return waitForNebula(ctx, env) + } + + for _, alloc := range allocs { adminAddr := net.JoinHostPort( env.Bootstrap.ThisHost().Nebula.IP, @@ -37,29 +44,6 @@ func waitForGarage(ctx context.Context, env crypticnet.Env) error { } -func waitForGarageArgs(env crypticnet.Env, args ...string) []string { - - thisHost := env.Bootstrap.ThisHost() - allocs := env.ThisDaemon().Storage.Allocations - - if len(allocs) == 0 { - return waitForNebulaArgs(env, args...) - } - - var preArgs []string - - for _, alloc := range allocs { - preArgs = append( - preArgs, - "wait-for", - net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)), - "--", - ) - } - - return append(preArgs, args...) -} - func garageWriteChildConf( env crypticnet.Env, alloc crypticnet.DaemonYmlStorageAllocation, @@ -116,7 +100,7 @@ func garageWriteChildConf( return garageTomlPath, nil } -func garageChildrenPmuxProcConfigs(env crypticnet.Env) ([]pmuxlib.ProcessConfig, error) { +func garagePmuxProcConfigs(env crypticnet.Env) ([]pmuxlib.ProcessConfig, error) { var pmuxProcConfigs []pmuxlib.ProcessConfig @@ -129,25 +113,18 @@ func garageChildrenPmuxProcConfigs(env crypticnet.Env) ([]pmuxlib.ProcessConfig, } pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{ - Name: fmt.Sprintf("garage-%d", alloc.RPCPort), - Cmd: "garage", - Args: []string{"-c", childConfPath, "server"}, - SigKillWait: 1 * time.Minute, + Name: fmt.Sprintf("garage-%d", alloc.RPCPort), + Cmd: "garage", + Args: []string{"-c", childConfPath, "server"}, + StartAfterFunc: func(ctx context.Context) error { + return waitForNebula(ctx, env) + }, }) } return pmuxProcConfigs, nil } -func garageApplyLayoutDiffPmuxProcConfig(env crypticnet.Env) pmuxlib.ProcessConfig { - return pmuxlib.ProcessConfig{ - Name: "garage-apply-layout-diff", - Cmd: "bash", - Args: waitForGarageArgs(env, "bash", "garage-apply-layout-diff"), - NoRestartOn: []int{0}, - } -} - func garageInitializeGlobalBucket(ctx context.Context, env crypticnet.Env) error { var ( diff --git a/go-workspace/src/cmd/entrypoint/hosts.go b/go-workspace/src/cmd/entrypoint/hosts.go index 68f3559..6fac6f9 100644 --- a/go-workspace/src/cmd/entrypoint/hosts.go +++ b/go-workspace/src/cmd/entrypoint/hosts.go @@ -1,4 +1,4 @@ -package entrypoint +package main import ( "cryptic-net/bootstrap" diff --git a/go-workspace/src/cmd/entrypoint/main.go b/go-workspace/src/cmd/entrypoint/main.go index ad6e254..a54f4ce 100644 --- a/go-workspace/src/cmd/entrypoint/main.go +++ b/go-workspace/src/cmd/entrypoint/main.go @@ -1,4 +1,4 @@ -package entrypoint +package main import ( "fmt" @@ -12,7 +12,7 @@ import ( // then passes execution along to an appropriate binary housed in AppDir/bin // (usually a bash script, which is more versatile than a go program). -func Main() { +func main() { env, err := crypticnet.NewEnv(true) diff --git a/go-workspace/src/cmd/nebula-entrypoint/main.go b/go-workspace/src/cmd/entrypoint/nebula_util.go similarity index 64% rename from go-workspace/src/cmd/nebula-entrypoint/main.go rename to go-workspace/src/cmd/entrypoint/nebula_util.go index 949f85e..c6737dc 100644 --- a/go-workspace/src/cmd/nebula-entrypoint/main.go +++ b/go-workspace/src/cmd/entrypoint/nebula_util.go @@ -1,24 +1,40 @@ -package nebula_entrypoint +package main import ( + "context" + crypticnet "cryptic-net" "cryptic-net/yamlutil" - "log" + "fmt" "net" - "os" "path/filepath" "strconv" - "syscall" - crypticnet "cryptic-net" + "code.betamike.com/cryptic-io/pmux/pmuxlib" ) -func Main() { +// waitForNebula waits for the nebula interface to have been started up. It does +// this by attempting to create a UDP connection which has the nebula IP set as +// its source. If this succeeds we can assume that at the very least the nebula +// interface has been initialized. +func waitForNebula(ctx context.Context, env crypticnet.Env) error { - env, err := crypticnet.ReadEnv() + ipStr := env.Bootstrap.ThisHost().Nebula.IP + ip := net.ParseIP(ipStr) - if err != nil { - log.Fatalf("reading envvars: %v", err) - } + lUdpAddr := &net.UDPAddr{IP: ip, Port: 0} + rUdpAddr := &net.UDPAddr{IP: ip, Port: 45535} + + return doOnce(ctx, func(context.Context) error { + conn, err := net.DialUDP("udp", lUdpAddr, rUdpAddr) + if err != nil { + return err + } + conn.Close() + return nil + }) +} + +func nebulaPmuxProcConfig(env crypticnet.Env) (pmuxlib.ProcessConfig, error) { var ( lighthouseHostIPs []string @@ -51,10 +67,6 @@ func Main() { }, } - if err != nil { - log.Fatal(err) - } - if publicAddr := env.ThisDaemon().VPN.PublicAddr; publicAddr == "" { config["listen"] = map[string]string{ @@ -71,7 +83,7 @@ func Main() { _, port, err := net.SplitHostPort(publicAddr) if err != nil { - log.Fatalf("parsing public address %q: %v", publicAddr, err) + return pmuxlib.ProcessConfig{}, fmt.Errorf("parsing public address %q: %w", publicAddr, err) } config["listen"] = map[string]string{ @@ -114,16 +126,12 @@ func Main() { nebulaYmlPath := filepath.Join(env.RuntimeDirPath, "nebula.yml") if err := yamlutil.WriteYamlFile(config, nebulaYmlPath); err != nil { - log.Fatalf("writing nebula.yml to %q: %v", nebulaYmlPath, err) + return pmuxlib.ProcessConfig{}, fmt.Errorf("writing nebula.yml to %q: %w", nebulaYmlPath, err) } - var ( - binPath = env.BinPath("nebula") - args = []string{"nebula", "-config", nebulaYmlPath} - cliEnv = os.Environ() - ) - - if err := syscall.Exec(binPath, args, cliEnv); err != nil { - log.Fatalf("calling exec(%q, %#v, %#v)", binPath, args, cliEnv) - } + return pmuxlib.ProcessConfig{ + Name: "nebula", + Cmd: "nebula", + Args: []string{"-config", nebulaYmlPath}, + }, nil } diff --git a/go-workspace/src/cmd/entrypoint/sub_cmd.go b/go-workspace/src/cmd/entrypoint/sub_cmd.go index dd53d61..cf4f21f 100644 --- a/go-workspace/src/cmd/entrypoint/sub_cmd.go +++ b/go-workspace/src/cmd/entrypoint/sub_cmd.go @@ -1,4 +1,4 @@ -package entrypoint +package main import ( crypticnet "cryptic-net" diff --git a/go-workspace/src/cmd/entrypoint/version.go b/go-workspace/src/cmd/entrypoint/version.go index f06af4e..1bab4e6 100644 --- a/go-workspace/src/cmd/entrypoint/version.go +++ b/go-workspace/src/cmd/entrypoint/version.go @@ -1,4 +1,4 @@ -package entrypoint +package main import ( "fmt"