Factor out nebula-entrypoint

As part of this all "wait" constraints have been migrated to pure-go
implementations, taking advantage of pmux's `StartAfterFunc` argument.

nebula-entrypoint was the final main process besides the entrypoint
itself, allowing us to get rid of cryptic-net-main.

AppDir/AppRun

@@ -1,4 +1,4 @@
 #!/bin/sh
 
 export PATH=$APPDIR/bin
-exec cryptic-net-main entrypoint "$@"
+exec entrypoint "$@"

default.nix

@@ -74,7 +74,7 @@
       dnsmasq
       garage
       waitFor
-      goWorkspace.crypticNetMain
+      goWorkspace.entrypoint
 
     ] ++ (if bootstrap != null then [ rootedBootstrap ] else []);
   };

go-workspace/default.nix

@@ -14,5 +14,5 @@
   };
 
 in {
-  crypticNetMain = build "cmd/cryptic-net-main";
+  entrypoint = build "cmd/entrypoint";
 }

go-workspace/src/cmd/cryptic-net-main/main.go

@@ -1,73 +0,0 @@
-package main
-
-//
-// This binary acts as a wrapper around other programs which would otherwise
-// form their own binaries. We do this for two reasons:
-//
-// * Nix makes it difficult to determine which individuals binaries need to be
-// rebuilt upon changes, so it rebuilds all of them no matter what changed. This
-// makes development slow. By wrapping everything in a sinble binary we only
-// ever have to build that binary.
-//
-// * If we have N binaries total, then we have N copies of the go runtime in our
-// final AppImage. By bundling the binaries into a single one we can reduce the
-// number go runtime copies to 1.
-//
-
-import (
-	"cryptic-net/cmd/entrypoint"
-	nebula_entrypoint "cryptic-net/cmd/nebula-entrypoint"
-	"fmt"
-	"os"
-)
-
-type mainFn struct {
-	name string
-	fn   func()
-}
-
-var mainFns = []mainFn{
-	{"entrypoint", entrypoint.Main},
-	{"nebula-entrypoint", nebula_entrypoint.Main},
-}
-
-var mainFnsMap = func() map[string]mainFn {
-
-	m := map[string]mainFn{}
-
-	for _, mainFn := range mainFns {
-		m[mainFn.name] = mainFn
-	}
-
-	return m
-}()
-
-func usage() {
-	fmt.Fprintf(os.Stderr, "USAGE: %s <cmd>\n\n", os.Args[0])
-	fmt.Fprintf(os.Stderr, "Commands:\n\n")
-
-	for _, mainFn := range mainFns {
-		fmt.Fprintf(os.Stderr, "%s\n", mainFn.name)
-	}
-
-	os.Stderr.Sync()
-	os.Exit(1)
-}
-
-func main() {
-
-	if len(os.Args) < 2 {
-		usage()
-	}
-
-	mainFn, ok := mainFnsMap[os.Args[1]]
-	if !ok {
-		usage()
-	}
-
-	// remove os.Args[1] from the arg list, so that other commands which consume
-	// args don't get confused
-	os.Args = append(os.Args[:1], os.Args[2:]...)
-
-	mainFn.fn()
-}

go-workspace/src/cmd/entrypoint/admin.go

@@ -1,4 +1,4 @@
-package entrypoint
+package main
 
 import (
 	"context"
@@ -173,17 +173,22 @@ var subCmdAdminCreateNetwork = subCmd{
 			}
 		}
 
-		garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env)
+		nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(env)
 		if err != nil {
-			return fmt.Errorf("generating garage children configs: %w", err)
+			return fmt.Errorf("generating nebula config: %w", err)
+		}
+
+		garagePmuxProcConfigs, err := garagePmuxProcConfigs(env)
+		if err != nil {
+			return fmt.Errorf("generating garage configs: %w", err)
 		}
 
 		pmuxConfig := pmuxlib.Config{
 			Processes: append(
 				[]pmuxlib.ProcessConfig{
-					nebulaEntrypointPmuxProcConfig(),
+					nebulaPmuxProcConfig,
 				},
-				garageChildrenPmuxProcConfigs...,
+				garagePmuxProcConfigs...,
 			),
 		}
 
@@ -203,7 +208,7 @@ var subCmdAdminCreateNetwork = subCmd{
 		}()
 
 		fmt.Fprintln(os.Stderr, "waiting for garage instances to come online")
-		if err := waitForGarage(ctx, env); err != nil {
+		if err := waitForGarageAndNebula(ctx, env); err != nil {
 			return fmt.Errorf("waiting for garage to start up: %w", err)
 		}
 

go-workspace/src/cmd/entrypoint/daemon.go

@@ -1,4 +1,4 @@
-package entrypoint
+package main
 
 import (
 	"bytes"
@@ -74,33 +74,46 @@ func reloadBootstrap(env crypticnet.Env, s3Client garage.S3APIClient) (crypticne
 	return env, true, nil
 }
 
-// runs a single pmux process ofor daemon, returning only once the env.Context
-// has been canceled or bootstrap info has been changed. This will always block
+// runs a single pmux process of daemon, returning only once the env.Context has
+// been canceled or bootstrap info has been changed. This will always block
 // until the spawned pmux has returned, and returns a copy of Env with updated
 // boostrap info.
-func runDaemonPmuxOnce(env crypticnet.Env, s3Client garage.S3APIClient) (crypticnet.Env, error) {
+func runDaemonPmuxOnce(env crypticnet.Env) (crypticnet.Env, error) {
 
 	thisHost := env.Bootstrap.ThisHost()
 	thisDaemon := env.ThisDaemon()
 	fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP)
 
+	// create s3Client anew on every loop, in case the topology has
+	// changed and we should be connecting to a different garage
+	// endpoint.
+	s3Client := env.Bootstrap.GlobalBucketS3APIClient()
+
+	nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(env)
+	if err != nil {
+		return crypticnet.Env{}, fmt.Errorf("generating nebula config: %w", err)
+	}
+
 	pmuxProcConfigs := []pmuxlib.ProcessConfig{
-		nebulaEntrypointPmuxProcConfig(),
+		nebulaPmuxProcConfig,
 		{
 			Name: "dnsmasq",
 			Cmd:  "bash",
-			Args: waitForNebulaArgs(env, "dnsmasq-entrypoint"),
+			Args: []string{"dnsmasq-entrypoint"},
+			StartAfterFunc: func(ctx context.Context) error {
+				return waitForNebula(ctx, env)
+			},
 		},
 	}
 
 	if len(thisDaemon.Storage.Allocations) > 0 {
 
-		garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env)
+		garagePmuxProcConfigs, err := garagePmuxProcConfigs(env)
 		if err != nil {
 			return crypticnet.Env{}, fmt.Errorf("generating garage children configs: %w", err)
 		}
 
-		pmuxProcConfigs = append(pmuxProcConfigs, garageChildrenPmuxProcConfigs...)
+		pmuxProcConfigs = append(pmuxProcConfigs, garagePmuxProcConfigs...)
 	}
 
 	pmuxConfig := pmuxlib.Config{Processes: pmuxProcConfigs}
@@ -123,14 +136,16 @@ func runDaemonPmuxOnce(env crypticnet.Env, s3Client garage.S3APIClient) (cryptic
 	go func() {
 		defer wg.Done()
 
-		// TODO wait for garage or nebula, depending on if allocs are present
+		if err := waitForGarageAndNebula(ctx, env); err != nil {
+			fmt.Fprintf(os.Stderr, "aborted waiting for garage instances to be accessible: %v\n", err)
+			return
+		}
 
-		client := env.Bootstrap.GlobalBucketS3APIClient()
 		thisHost := env.Bootstrap.ThisHost()
 
 		err := doOnce(ctx, func(ctx context.Context) error {
 			fmt.Fprintln(os.Stderr, "updating host info in garage")
-			return bootstrap.PutGarageBoostrapHost(ctx, client, thisHost)
+			return bootstrap.PutGarageBoostrapHost(ctx, s3Client, thisHost)
 		})
 
 		if err != nil {
@@ -143,8 +158,8 @@ func runDaemonPmuxOnce(env crypticnet.Env, s3Client garage.S3APIClient) (cryptic
 		go func() {
 			defer wg.Done()
 
-			if err := waitForGarage(ctx, env); err != nil {
-				fmt.Fprintf(os.Stderr, "aborted waiting for garage instances to start: %v\n", err)
+			if err := waitForGarageAndNebula(ctx, env); err != nil {
+				fmt.Fprintf(os.Stderr, "aborted waiting for garage instances to be accessible: %v\n", err)
 				return
 			}
 
@@ -287,6 +302,9 @@ var subCmdDaemon = subCmd{
 			return fmt.Errorf("merging daemon.yml into bootstrap data: %w", err)
 		}
 
+		// TODO once dnsmasq entrypoint is written in go and the config
+		// generation is inlined into this process then this Setenv won't be
+		// necessary.
 		for key, val := range env.ToMap() {
 			if err := os.Setenv(key, val); err != nil {
 				return fmt.Errorf("failed to set %q to %q: %w", key, val, err)
@@ -295,12 +313,7 @@ var subCmdDaemon = subCmd{
 
 		for {
 
-			// create s3Client anew on every loop, in case the topology has
-			// changed and we should be connecting to a different garage
-			// endpoint.
-			s3Client := env.Bootstrap.GlobalBucketS3APIClient()
-
-			if env, err = runDaemonPmuxOnce(env, s3Client); errors.Is(err, context.Canceled) {
+			if env, err = runDaemonPmuxOnce(env); errors.Is(err, context.Canceled) {
 				return nil
 
 			} else if err != nil {

go-workspace/src/cmd/entrypoint/daemon_util.go

@@ -1,4 +1,4 @@
-package entrypoint
+package main
 
 import (
 	"bytes"
@@ -9,8 +9,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-
-	"code.betamike.com/cryptic-io/pmux/pmuxlib"
+	"time"
 )
 
 func copyBootstrapToDataDirAndReload(env crypticnet.Env, r io.Reader) (crypticnet.Env, error) {
@@ -74,20 +73,7 @@ func doOnce(ctx context.Context, fn func(context.Context) error) error {
 		} else if ctxErr := ctx.Err(); ctxErr != nil {
 			return ctxErr
 		}
-	}
-}
-
-func waitForNebulaArgs(env crypticnet.Env, args ...string) []string {
-	thisHost := env.Bootstrap.ThisHost()
-	return append([]string{"wait-for-ip", thisHost.Nebula.IP}, args...)
-}
-
-func nebulaEntrypointPmuxProcConfig() pmuxlib.ProcessConfig {
-	return pmuxlib.ProcessConfig{
-		Name: "nebula",
-		Cmd:  "cryptic-net-main",
-		Args: []string{
-			"nebula-entrypoint",
-		},
+
+		time.Sleep(250 * time.Millisecond)
 	}
 }

go-workspace/src/cmd/entrypoint/daemon_yml.go

@@ -1,4 +1,4 @@
-package entrypoint
+package main
 
 import (
 	crypticnet "cryptic-net"

go-workspace/src/cmd/entrypoint/garage.go

@@ -1,4 +1,4 @@
-package entrypoint
+package main
 
 import (
 	"fmt"

go-workspace/src/cmd/entrypoint/garage_util.go

@@ -1,4 +1,4 @@
-package entrypoint
+package main
 
 import (
 	"context"
@@ -9,14 +9,21 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
-	"time"
 
 	"code.betamike.com/cryptic-io/pmux/pmuxlib"
 )
 
-func waitForGarage(ctx context.Context, env crypticnet.Env) error {
+func waitForGarageAndNebula(ctx context.Context, env crypticnet.Env) error {
 
-	for _, alloc := range env.ThisDaemon().Storage.Allocations {
+	allocs := env.ThisDaemon().Storage.Allocations
+
+	// if this host doesn't have any allocations specified then fall back to
+	// waiting for nebula
+	if len(allocs) == 0 {
+		return waitForNebula(ctx, env)
+	}
+
+	for _, alloc := range allocs {
 
 		adminAddr := net.JoinHostPort(
 			env.Bootstrap.ThisHost().Nebula.IP,
@@ -37,29 +44,6 @@ func waitForGarage(ctx context.Context, env crypticnet.Env) error {
 
 }
 
-func waitForGarageArgs(env crypticnet.Env, args ...string) []string {
-
-	thisHost := env.Bootstrap.ThisHost()
-	allocs := env.ThisDaemon().Storage.Allocations
-
-	if len(allocs) == 0 {
-		return waitForNebulaArgs(env, args...)
-	}
-
-	var preArgs []string
-
-	for _, alloc := range allocs {
-		preArgs = append(
-			preArgs,
-			"wait-for",
-			net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
-			"--",
-		)
-	}
-
-	return append(preArgs, args...)
-}
-
 func garageWriteChildConf(
 	env crypticnet.Env,
 	alloc crypticnet.DaemonYmlStorageAllocation,
@@ -116,7 +100,7 @@ func garageWriteChildConf(
 	return garageTomlPath, nil
 }
 
-func garageChildrenPmuxProcConfigs(env crypticnet.Env) ([]pmuxlib.ProcessConfig, error) {
+func garagePmuxProcConfigs(env crypticnet.Env) ([]pmuxlib.ProcessConfig, error) {
 
 	var pmuxProcConfigs []pmuxlib.ProcessConfig
 
@@ -132,22 +116,15 @@ func garageChildrenPmuxProcConfigs(env crypticnet.Env) ([]pmuxlib.ProcessConfig,
 			Name: fmt.Sprintf("garage-%d", alloc.RPCPort),
 			Cmd:  "garage",
 			Args: []string{"-c", childConfPath, "server"},
-			SigKillWait: 1 * time.Minute,
+			StartAfterFunc: func(ctx context.Context) error {
+				return waitForNebula(ctx, env)
+			},
 		})
 	}
 
 	return pmuxProcConfigs, nil
 }
 
-func garageApplyLayoutDiffPmuxProcConfig(env crypticnet.Env) pmuxlib.ProcessConfig {
-	return pmuxlib.ProcessConfig{
-		Name:        "garage-apply-layout-diff",
-		Cmd:         "bash",
-		Args:        waitForGarageArgs(env, "bash", "garage-apply-layout-diff"),
-		NoRestartOn: []int{0},
-	}
-}
-
 func garageInitializeGlobalBucket(ctx context.Context, env crypticnet.Env) error {
 
 	var (

go-workspace/src/cmd/entrypoint/hosts.go

@@ -1,4 +1,4 @@
-package entrypoint
+package main
 
 import (
 	"cryptic-net/bootstrap"

go-workspace/src/cmd/entrypoint/main.go

@@ -1,4 +1,4 @@
-package entrypoint
+package main
 
 import (
 	"fmt"
@@ -12,7 +12,7 @@ import (
 // then passes execution along to an appropriate binary housed in AppDir/bin
 // (usually a bash script, which is more versatile than a go program).
 
-func Main() {
+func main() {
 
 	env, err := crypticnet.NewEnv(true)
 

go-workspace/src/cmd/entrypoint/nebula_util.go

@@ -1,24 +1,40 @@
-package nebula_entrypoint
+package main
 
 import (
+	"context"
+	crypticnet "cryptic-net"
 	"cryptic-net/yamlutil"
-	"log"
+	"fmt"
 	"net"
-	"os"
 	"path/filepath"
 	"strconv"
-	"syscall"
 
-	crypticnet "cryptic-net"
+	"code.betamike.com/cryptic-io/pmux/pmuxlib"
 )
 
-func Main() {
+// waitForNebula waits for the nebula interface to have been started up. It does
+// this by attempting to create a UDP connection which has the nebula IP set as
+// its source. If this succeeds we can assume that at the very least the nebula
+// interface has been initialized.
+func waitForNebula(ctx context.Context, env crypticnet.Env) error {
 
-	env, err := crypticnet.ReadEnv()
+	ipStr := env.Bootstrap.ThisHost().Nebula.IP
+	ip := net.ParseIP(ipStr)
 
+	lUdpAddr := &net.UDPAddr{IP: ip, Port: 0}
+	rUdpAddr := &net.UDPAddr{IP: ip, Port: 45535}
+
+	return doOnce(ctx, func(context.Context) error {
+		conn, err := net.DialUDP("udp", lUdpAddr, rUdpAddr)
 		if err != nil {
-		log.Fatalf("reading envvars: %v", err)
+			return err
 		}
+		conn.Close()
+		return nil
+	})
+}
+
+func nebulaPmuxProcConfig(env crypticnet.Env) (pmuxlib.ProcessConfig, error) {
 
 	var (
 		lighthouseHostIPs []string
@@ -51,10 +67,6 @@ func Main() {
 		},
 	}
 
-	if err != nil {
-		log.Fatal(err)
-	}
-
 	if publicAddr := env.ThisDaemon().VPN.PublicAddr; publicAddr == "" {
 
 		config["listen"] = map[string]string{
@@ -71,7 +83,7 @@ func Main() {
 		_, port, err := net.SplitHostPort(publicAddr)
 
 		if err != nil {
-			log.Fatalf("parsing public address %q: %v", publicAddr, err)
+			return pmuxlib.ProcessConfig{}, fmt.Errorf("parsing public address %q: %w", publicAddr, err)
 		}
 
 		config["listen"] = map[string]string{
@@ -114,16 +126,12 @@ func Main() {
 	nebulaYmlPath := filepath.Join(env.RuntimeDirPath, "nebula.yml")
 
 	if err := yamlutil.WriteYamlFile(config, nebulaYmlPath); err != nil {
-		log.Fatalf("writing nebula.yml to %q: %v", nebulaYmlPath, err)
+		return pmuxlib.ProcessConfig{}, fmt.Errorf("writing nebula.yml to %q: %w", nebulaYmlPath, err)
 	}
 
-	var (
-		binPath = env.BinPath("nebula")
-		args    = []string{"nebula", "-config", nebulaYmlPath}
-		cliEnv  = os.Environ()
-	)
-
-	if err := syscall.Exec(binPath, args, cliEnv); err != nil {
-		log.Fatalf("calling exec(%q, %#v, %#v)", binPath, args, cliEnv)
-	}
+	return pmuxlib.ProcessConfig{
+		Name: "nebula",
+		Cmd:  "nebula",
+		Args: []string{"-config", nebulaYmlPath},
+	}, nil
 }

go-workspace/src/cmd/entrypoint/sub_cmd.go

@@ -1,4 +1,4 @@
-package entrypoint
+package main
 
 import (
 	crypticnet "cryptic-net"

go-workspace/src/cmd/entrypoint/version.go

@@ -1,4 +1,4 @@
-package entrypoint
+package main
 
 import (
 	"fmt"