Factor out nebula-entrypoint

As part of this all "wait" constraints have been migrated to pure-go
implementations, taking advantage of pmux's `StartAfterFunc` argument.

nebula-entrypoint was the final main process besides the entrypoint
itself, allowing us to get rid of cryptic-net-main.
This commit is contained in:
Brian Picciano 2022-10-20 21:59:46 +02:00
parent 8ba88b4dfc
commit 47e45e0071
15 changed files with 108 additions and 192 deletions

View File

@ -1,4 +1,4 @@
#!/bin/sh #!/bin/sh
export PATH=$APPDIR/bin export PATH=$APPDIR/bin
exec cryptic-net-main entrypoint "$@" exec entrypoint "$@"

View File

@ -74,7 +74,7 @@
dnsmasq dnsmasq
garage garage
waitFor waitFor
goWorkspace.crypticNetMain goWorkspace.entrypoint
] ++ (if bootstrap != null then [ rootedBootstrap ] else []); ] ++ (if bootstrap != null then [ rootedBootstrap ] else []);
}; };

View File

@ -14,5 +14,5 @@
}; };
in { in {
crypticNetMain = build "cmd/cryptic-net-main"; entrypoint = build "cmd/entrypoint";
} }

View File

@ -1,73 +0,0 @@
package main
//
// This binary acts as a wrapper around other programs which would otherwise
// form their own binaries. We do this for two reasons:
//
// * Nix makes it difficult to determine which individuals binaries need to be
// rebuilt upon changes, so it rebuilds all of them no matter what changed. This
// makes development slow. By wrapping everything in a sinble binary we only
// ever have to build that binary.
//
// * If we have N binaries total, then we have N copies of the go runtime in our
// final AppImage. By bundling the binaries into a single one we can reduce the
// number go runtime copies to 1.
//
import (
"cryptic-net/cmd/entrypoint"
nebula_entrypoint "cryptic-net/cmd/nebula-entrypoint"
"fmt"
"os"
)
type mainFn struct {
name string
fn func()
}
var mainFns = []mainFn{
{"entrypoint", entrypoint.Main},
{"nebula-entrypoint", nebula_entrypoint.Main},
}
var mainFnsMap = func() map[string]mainFn {
m := map[string]mainFn{}
for _, mainFn := range mainFns {
m[mainFn.name] = mainFn
}
return m
}()
func usage() {
fmt.Fprintf(os.Stderr, "USAGE: %s <cmd>\n\n", os.Args[0])
fmt.Fprintf(os.Stderr, "Commands:\n\n")
for _, mainFn := range mainFns {
fmt.Fprintf(os.Stderr, "%s\n", mainFn.name)
}
os.Stderr.Sync()
os.Exit(1)
}
func main() {
if len(os.Args) < 2 {
usage()
}
mainFn, ok := mainFnsMap[os.Args[1]]
if !ok {
usage()
}
// remove os.Args[1] from the arg list, so that other commands which consume
// args don't get confused
os.Args = append(os.Args[:1], os.Args[2:]...)
mainFn.fn()
}

View File

@ -1,4 +1,4 @@
package entrypoint package main
import ( import (
"context" "context"
@ -173,17 +173,22 @@ var subCmdAdminCreateNetwork = subCmd{
} }
} }
garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env) nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(env)
if err != nil { if err != nil {
return fmt.Errorf("generating garage children configs: %w", err) return fmt.Errorf("generating nebula config: %w", err)
}
garagePmuxProcConfigs, err := garagePmuxProcConfigs(env)
if err != nil {
return fmt.Errorf("generating garage configs: %w", err)
} }
pmuxConfig := pmuxlib.Config{ pmuxConfig := pmuxlib.Config{
Processes: append( Processes: append(
[]pmuxlib.ProcessConfig{ []pmuxlib.ProcessConfig{
nebulaEntrypointPmuxProcConfig(), nebulaPmuxProcConfig,
}, },
garageChildrenPmuxProcConfigs..., garagePmuxProcConfigs...,
), ),
} }
@ -203,7 +208,7 @@ var subCmdAdminCreateNetwork = subCmd{
}() }()
fmt.Fprintln(os.Stderr, "waiting for garage instances to come online") fmt.Fprintln(os.Stderr, "waiting for garage instances to come online")
if err := waitForGarage(ctx, env); err != nil { if err := waitForGarageAndNebula(ctx, env); err != nil {
return fmt.Errorf("waiting for garage to start up: %w", err) return fmt.Errorf("waiting for garage to start up: %w", err)
} }

View File

@ -1,4 +1,4 @@
package entrypoint package main
import ( import (
"bytes" "bytes"
@ -74,33 +74,46 @@ func reloadBootstrap(env crypticnet.Env, s3Client garage.S3APIClient) (crypticne
return env, true, nil return env, true, nil
} }
// runs a single pmux process ofor daemon, returning only once the env.Context // runs a single pmux process of daemon, returning only once the env.Context has
// has been canceled or bootstrap info has been changed. This will always block // been canceled or bootstrap info has been changed. This will always block
// until the spawned pmux has returned, and returns a copy of Env with updated // until the spawned pmux has returned, and returns a copy of Env with updated
// boostrap info. // boostrap info.
func runDaemonPmuxOnce(env crypticnet.Env, s3Client garage.S3APIClient) (crypticnet.Env, error) { func runDaemonPmuxOnce(env crypticnet.Env) (crypticnet.Env, error) {
thisHost := env.Bootstrap.ThisHost() thisHost := env.Bootstrap.ThisHost()
thisDaemon := env.ThisDaemon() thisDaemon := env.ThisDaemon()
fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP) fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP)
// create s3Client anew on every loop, in case the topology has
// changed and we should be connecting to a different garage
// endpoint.
s3Client := env.Bootstrap.GlobalBucketS3APIClient()
nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(env)
if err != nil {
return crypticnet.Env{}, fmt.Errorf("generating nebula config: %w", err)
}
pmuxProcConfigs := []pmuxlib.ProcessConfig{ pmuxProcConfigs := []pmuxlib.ProcessConfig{
nebulaEntrypointPmuxProcConfig(), nebulaPmuxProcConfig,
{ {
Name: "dnsmasq", Name: "dnsmasq",
Cmd: "bash", Cmd: "bash",
Args: waitForNebulaArgs(env, "dnsmasq-entrypoint"), Args: []string{"dnsmasq-entrypoint"},
StartAfterFunc: func(ctx context.Context) error {
return waitForNebula(ctx, env)
},
}, },
} }
if len(thisDaemon.Storage.Allocations) > 0 { if len(thisDaemon.Storage.Allocations) > 0 {
garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env) garagePmuxProcConfigs, err := garagePmuxProcConfigs(env)
if err != nil { if err != nil {
return crypticnet.Env{}, fmt.Errorf("generating garage children configs: %w", err) return crypticnet.Env{}, fmt.Errorf("generating garage children configs: %w", err)
} }
pmuxProcConfigs = append(pmuxProcConfigs, garageChildrenPmuxProcConfigs...) pmuxProcConfigs = append(pmuxProcConfigs, garagePmuxProcConfigs...)
} }
pmuxConfig := pmuxlib.Config{Processes: pmuxProcConfigs} pmuxConfig := pmuxlib.Config{Processes: pmuxProcConfigs}
@ -123,14 +136,16 @@ func runDaemonPmuxOnce(env crypticnet.Env, s3Client garage.S3APIClient) (cryptic
go func() { go func() {
defer wg.Done() defer wg.Done()
// TODO wait for garage or nebula, depending on if allocs are present if err := waitForGarageAndNebula(ctx, env); err != nil {
fmt.Fprintf(os.Stderr, "aborted waiting for garage instances to be accessible: %v\n", err)
return
}
client := env.Bootstrap.GlobalBucketS3APIClient()
thisHost := env.Bootstrap.ThisHost() thisHost := env.Bootstrap.ThisHost()
err := doOnce(ctx, func(ctx context.Context) error { err := doOnce(ctx, func(ctx context.Context) error {
fmt.Fprintln(os.Stderr, "updating host info in garage") fmt.Fprintln(os.Stderr, "updating host info in garage")
return bootstrap.PutGarageBoostrapHost(ctx, client, thisHost) return bootstrap.PutGarageBoostrapHost(ctx, s3Client, thisHost)
}) })
if err != nil { if err != nil {
@ -143,8 +158,8 @@ func runDaemonPmuxOnce(env crypticnet.Env, s3Client garage.S3APIClient) (cryptic
go func() { go func() {
defer wg.Done() defer wg.Done()
if err := waitForGarage(ctx, env); err != nil { if err := waitForGarageAndNebula(ctx, env); err != nil {
fmt.Fprintf(os.Stderr, "aborted waiting for garage instances to start: %v\n", err) fmt.Fprintf(os.Stderr, "aborted waiting for garage instances to be accessible: %v\n", err)
return return
} }
@ -287,6 +302,9 @@ var subCmdDaemon = subCmd{
return fmt.Errorf("merging daemon.yml into bootstrap data: %w", err) return fmt.Errorf("merging daemon.yml into bootstrap data: %w", err)
} }
// TODO once dnsmasq entrypoint is written in go and the config
// generation is inlined into this process then this Setenv won't be
// necessary.
for key, val := range env.ToMap() { for key, val := range env.ToMap() {
if err := os.Setenv(key, val); err != nil { if err := os.Setenv(key, val); err != nil {
return fmt.Errorf("failed to set %q to %q: %w", key, val, err) return fmt.Errorf("failed to set %q to %q: %w", key, val, err)
@ -295,12 +313,7 @@ var subCmdDaemon = subCmd{
for { for {
// create s3Client anew on every loop, in case the topology has if env, err = runDaemonPmuxOnce(env); errors.Is(err, context.Canceled) {
// changed and we should be connecting to a different garage
// endpoint.
s3Client := env.Bootstrap.GlobalBucketS3APIClient()
if env, err = runDaemonPmuxOnce(env, s3Client); errors.Is(err, context.Canceled) {
return nil return nil
} else if err != nil { } else if err != nil {

View File

@ -1,4 +1,4 @@
package entrypoint package main
import ( import (
"bytes" "bytes"
@ -9,8 +9,7 @@ import (
"io" "io"
"os" "os"
"path/filepath" "path/filepath"
"time"
"code.betamike.com/cryptic-io/pmux/pmuxlib"
) )
func copyBootstrapToDataDirAndReload(env crypticnet.Env, r io.Reader) (crypticnet.Env, error) { func copyBootstrapToDataDirAndReload(env crypticnet.Env, r io.Reader) (crypticnet.Env, error) {
@ -74,20 +73,7 @@ func doOnce(ctx context.Context, fn func(context.Context) error) error {
} else if ctxErr := ctx.Err(); ctxErr != nil { } else if ctxErr := ctx.Err(); ctxErr != nil {
return ctxErr return ctxErr
} }
}
} time.Sleep(250 * time.Millisecond)
func waitForNebulaArgs(env crypticnet.Env, args ...string) []string {
thisHost := env.Bootstrap.ThisHost()
return append([]string{"wait-for-ip", thisHost.Nebula.IP}, args...)
}
func nebulaEntrypointPmuxProcConfig() pmuxlib.ProcessConfig {
return pmuxlib.ProcessConfig{
Name: "nebula",
Cmd: "cryptic-net-main",
Args: []string{
"nebula-entrypoint",
},
} }
} }

View File

@ -1,4 +1,4 @@
package entrypoint package main
import ( import (
crypticnet "cryptic-net" crypticnet "cryptic-net"

View File

@ -1,4 +1,4 @@
package entrypoint package main
import ( import (
"fmt" "fmt"

View File

@ -1,4 +1,4 @@
package entrypoint package main
import ( import (
"context" "context"
@ -9,14 +9,21 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strconv" "strconv"
"time"
"code.betamike.com/cryptic-io/pmux/pmuxlib" "code.betamike.com/cryptic-io/pmux/pmuxlib"
) )
func waitForGarage(ctx context.Context, env crypticnet.Env) error { func waitForGarageAndNebula(ctx context.Context, env crypticnet.Env) error {
for _, alloc := range env.ThisDaemon().Storage.Allocations { allocs := env.ThisDaemon().Storage.Allocations
// if this host doesn't have any allocations specified then fall back to
// waiting for nebula
if len(allocs) == 0 {
return waitForNebula(ctx, env)
}
for _, alloc := range allocs {
adminAddr := net.JoinHostPort( adminAddr := net.JoinHostPort(
env.Bootstrap.ThisHost().Nebula.IP, env.Bootstrap.ThisHost().Nebula.IP,
@ -37,29 +44,6 @@ func waitForGarage(ctx context.Context, env crypticnet.Env) error {
} }
func waitForGarageArgs(env crypticnet.Env, args ...string) []string {
thisHost := env.Bootstrap.ThisHost()
allocs := env.ThisDaemon().Storage.Allocations
if len(allocs) == 0 {
return waitForNebulaArgs(env, args...)
}
var preArgs []string
for _, alloc := range allocs {
preArgs = append(
preArgs,
"wait-for",
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
"--",
)
}
return append(preArgs, args...)
}
func garageWriteChildConf( func garageWriteChildConf(
env crypticnet.Env, env crypticnet.Env,
alloc crypticnet.DaemonYmlStorageAllocation, alloc crypticnet.DaemonYmlStorageAllocation,
@ -116,7 +100,7 @@ func garageWriteChildConf(
return garageTomlPath, nil return garageTomlPath, nil
} }
func garageChildrenPmuxProcConfigs(env crypticnet.Env) ([]pmuxlib.ProcessConfig, error) { func garagePmuxProcConfigs(env crypticnet.Env) ([]pmuxlib.ProcessConfig, error) {
var pmuxProcConfigs []pmuxlib.ProcessConfig var pmuxProcConfigs []pmuxlib.ProcessConfig
@ -129,25 +113,18 @@ func garageChildrenPmuxProcConfigs(env crypticnet.Env) ([]pmuxlib.ProcessConfig,
} }
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{ pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
Name: fmt.Sprintf("garage-%d", alloc.RPCPort), Name: fmt.Sprintf("garage-%d", alloc.RPCPort),
Cmd: "garage", Cmd: "garage",
Args: []string{"-c", childConfPath, "server"}, Args: []string{"-c", childConfPath, "server"},
SigKillWait: 1 * time.Minute, StartAfterFunc: func(ctx context.Context) error {
return waitForNebula(ctx, env)
},
}) })
} }
return pmuxProcConfigs, nil return pmuxProcConfigs, nil
} }
func garageApplyLayoutDiffPmuxProcConfig(env crypticnet.Env) pmuxlib.ProcessConfig {
return pmuxlib.ProcessConfig{
Name: "garage-apply-layout-diff",
Cmd: "bash",
Args: waitForGarageArgs(env, "bash", "garage-apply-layout-diff"),
NoRestartOn: []int{0},
}
}
func garageInitializeGlobalBucket(ctx context.Context, env crypticnet.Env) error { func garageInitializeGlobalBucket(ctx context.Context, env crypticnet.Env) error {
var ( var (

View File

@ -1,4 +1,4 @@
package entrypoint package main
import ( import (
"cryptic-net/bootstrap" "cryptic-net/bootstrap"

View File

@ -1,4 +1,4 @@
package entrypoint package main
import ( import (
"fmt" "fmt"
@ -12,7 +12,7 @@ import (
// then passes execution along to an appropriate binary housed in AppDir/bin // then passes execution along to an appropriate binary housed in AppDir/bin
// (usually a bash script, which is more versatile than a go program). // (usually a bash script, which is more versatile than a go program).
func Main() { func main() {
env, err := crypticnet.NewEnv(true) env, err := crypticnet.NewEnv(true)

View File

@ -1,24 +1,40 @@
package nebula_entrypoint package main
import ( import (
"context"
crypticnet "cryptic-net"
"cryptic-net/yamlutil" "cryptic-net/yamlutil"
"log" "fmt"
"net" "net"
"os"
"path/filepath" "path/filepath"
"strconv" "strconv"
"syscall"
crypticnet "cryptic-net" "code.betamike.com/cryptic-io/pmux/pmuxlib"
) )
func Main() { // waitForNebula waits for the nebula interface to have been started up. It does
// this by attempting to create a UDP connection which has the nebula IP set as
// its source. If this succeeds we can assume that at the very least the nebula
// interface has been initialized.
func waitForNebula(ctx context.Context, env crypticnet.Env) error {
env, err := crypticnet.ReadEnv() ipStr := env.Bootstrap.ThisHost().Nebula.IP
ip := net.ParseIP(ipStr)
if err != nil { lUdpAddr := &net.UDPAddr{IP: ip, Port: 0}
log.Fatalf("reading envvars: %v", err) rUdpAddr := &net.UDPAddr{IP: ip, Port: 45535}
}
return doOnce(ctx, func(context.Context) error {
conn, err := net.DialUDP("udp", lUdpAddr, rUdpAddr)
if err != nil {
return err
}
conn.Close()
return nil
})
}
func nebulaPmuxProcConfig(env crypticnet.Env) (pmuxlib.ProcessConfig, error) {
var ( var (
lighthouseHostIPs []string lighthouseHostIPs []string
@ -51,10 +67,6 @@ func Main() {
}, },
} }
if err != nil {
log.Fatal(err)
}
if publicAddr := env.ThisDaemon().VPN.PublicAddr; publicAddr == "" { if publicAddr := env.ThisDaemon().VPN.PublicAddr; publicAddr == "" {
config["listen"] = map[string]string{ config["listen"] = map[string]string{
@ -71,7 +83,7 @@ func Main() {
_, port, err := net.SplitHostPort(publicAddr) _, port, err := net.SplitHostPort(publicAddr)
if err != nil { if err != nil {
log.Fatalf("parsing public address %q: %v", publicAddr, err) return pmuxlib.ProcessConfig{}, fmt.Errorf("parsing public address %q: %w", publicAddr, err)
} }
config["listen"] = map[string]string{ config["listen"] = map[string]string{
@ -114,16 +126,12 @@ func Main() {
nebulaYmlPath := filepath.Join(env.RuntimeDirPath, "nebula.yml") nebulaYmlPath := filepath.Join(env.RuntimeDirPath, "nebula.yml")
if err := yamlutil.WriteYamlFile(config, nebulaYmlPath); err != nil { if err := yamlutil.WriteYamlFile(config, nebulaYmlPath); err != nil {
log.Fatalf("writing nebula.yml to %q: %v", nebulaYmlPath, err) return pmuxlib.ProcessConfig{}, fmt.Errorf("writing nebula.yml to %q: %w", nebulaYmlPath, err)
} }
var ( return pmuxlib.ProcessConfig{
binPath = env.BinPath("nebula") Name: "nebula",
args = []string{"nebula", "-config", nebulaYmlPath} Cmd: "nebula",
cliEnv = os.Environ() Args: []string{"-config", nebulaYmlPath},
) }, nil
if err := syscall.Exec(binPath, args, cliEnv); err != nil {
log.Fatalf("calling exec(%q, %#v, %#v)", binPath, args, cliEnv)
}
} }

View File

@ -1,4 +1,4 @@
package entrypoint package main
import ( import (
crypticnet "cryptic-net" crypticnet "cryptic-net"

View File

@ -1,4 +1,4 @@
package entrypoint package main
import ( import (
"fmt" "fmt"