Compare commits

...

3 Commits

Author SHA1 Message Date
Brian Picciano
9288d8cf48 Fix and improve version string 2022-10-20 22:30:30 +02:00
Brian Picciano
5e399209b2 Rename go-workspace to just entrypoint, clean out unused wait-for tools 2022-10-20 22:06:22 +02:00
Brian Picciano
47e45e0071 Factor out nebula-entrypoint
As part of this all "wait" constraints have been migrated to pure-go
implementations, taking advantage of pmux's `StartAfterFunc` argument.

nebula-entrypoint was the final main process besides the entrypoint
itself, allowing us to get rid of cryptic-net-main.
2022-10-20 21:59:46 +02:00
44 changed files with 147 additions and 262 deletions

View File

@ -1,4 +1,4 @@
#!/bin/sh
export PATH=$APPDIR/bin
exec cryptic-net-main entrypoint "$@"
exec entrypoint "$@"

View File

@ -1,9 +0,0 @@
ip="$1"
shift;
echo "waiting for $ip to become available..."
while true; do ping -c1 -W1 "$ip" &> /dev/null && break; done
exec "$@"

View File

@ -1,9 +1,13 @@
{
pkgs ? (import ./nix/pkgs.nix).stable,
pkgsAttrs ? (import ./nix/pkgs.nix),
bootstrap ? null,
}: rec {
}: let
pkgs = pkgsAttrs.pkgs;
in rec {
rootedBootstrap = pkgs.stdenv.mkDerivation {
name = "cryptic-net-rooted-bootstrap";
@ -21,30 +25,37 @@
name = "cryptic-net-version";
buildInputs = [ pkgs.git pkgs.go ];
src = ./.;
inherit bootstrap;
nixPkgsVersion = pkgsAttrs.version;
nixPkgsRev = pkgsAttrs.rev;
builtByUser = builtins.getEnv "USER";
builder = builtins.toFile "builder.sh" ''
source $stdenv/setup
versionFile=version
cp -r "$src" srcCp
if [ "$bootstrap" != "" ]; then
hostName=$(tar -xzf "$bootstrap" --to-stdout ./hostname)
echo "Built for host: $hostName" >> "$versionFile"
fi
echo "Build date: $(date)" >> "$versionFile"
echo "Git status: $(cd "$src" && git describe --always --long --dirty=' (dirty)')" >> "$versionFile"
echo "Build date: $(date) ($(date +%s))" >> "$versionFile"
echo "Built by: $builtByUser" >> "$versionFile"
echo "Git rev: $(cd srcCp && git describe --always --long --dirty=' (dirty)')" >> "$versionFile"
echo "Go version: $(go version)" >> "$versionFile"
echo "Build host info: $(uname -srvm)" >> "$versionFile"
echo "Nixpkgs version: $nixPkgsVersion ($nixPkgsRev)" >> "$versionFile"
mkdir -p "$out"/share
cp "$versionFile" "$out"/share
'';
};
goWorkspace = pkgs.callPackage ./go-workspace {};
entrypoint = pkgs.callPackage ./entrypoint {};
dnsmasq = (pkgs.callPackage ./dnsmasq {
glibcStatic = pkgs.glibc.static;
@ -60,8 +71,6 @@
pkgs.pkgsStatic.bash
pkgs.pkgsStatic.coreutils
pkgs.pkgsStatic.unixtools.ping
pkgs.pkgsStatic.netcat # required by waitFor
pkgs.pkgsStatic.gnutar
pkgs.pkgsStatic.gzip
@ -73,8 +82,7 @@
version
dnsmasq
garage
waitFor
goWorkspace.crypticNetMain
entrypoint
] ++ (if bootstrap != null then [ rootedBootstrap ] else []);
};

View File

@ -1,6 +1,6 @@
{
pkgs ? (import ../nix/pkgs.nix).stable,
pkgs ? (import ../nix/pkgs.nix).pkgs,
}: pkgs.mkShell {
name = "cryptic-net-build-docs";

14
entrypoint/default.nix Normal file
View File

@ -0,0 +1,14 @@
{
buildGoModule,
}: buildGoModule {
pname = "cryptic-net-entrypoint";
version = "unstable";
src = ./src;
vendorSha256 = "sha256-URmrK9Sd/5yhXrWxXZq05TS7aY7IWptQFMKfXKJY7Hc=";
subPackages = [
"cmd/entrypoint"
];
}

View File

@ -1,4 +1,4 @@
package entrypoint
package main
import (
"context"
@ -173,17 +173,22 @@ var subCmdAdminCreateNetwork = subCmd{
}
}
garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env)
nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(env)
if err != nil {
return fmt.Errorf("generating garage children configs: %w", err)
return fmt.Errorf("generating nebula config: %w", err)
}
garagePmuxProcConfigs, err := garagePmuxProcConfigs(env)
if err != nil {
return fmt.Errorf("generating garage configs: %w", err)
}
pmuxConfig := pmuxlib.Config{
Processes: append(
[]pmuxlib.ProcessConfig{
nebulaEntrypointPmuxProcConfig(),
nebulaPmuxProcConfig,
},
garageChildrenPmuxProcConfigs...,
garagePmuxProcConfigs...,
),
}
@ -203,7 +208,7 @@ var subCmdAdminCreateNetwork = subCmd{
}()
fmt.Fprintln(os.Stderr, "waiting for garage instances to come online")
if err := waitForGarage(ctx, env); err != nil {
if err := waitForGarageAndNebula(ctx, env); err != nil {
return fmt.Errorf("waiting for garage to start up: %w", err)
}

View File

@ -1,4 +1,4 @@
package entrypoint
package main
import (
"bytes"
@ -74,33 +74,46 @@ func reloadBootstrap(env crypticnet.Env, s3Client garage.S3APIClient) (crypticne
return env, true, nil
}
// runs a single pmux process ofor daemon, returning only once the env.Context
// has been canceled or bootstrap info has been changed. This will always block
// runs a single pmux process of daemon, returning only once the env.Context has
// been canceled or bootstrap info has been changed. This will always block
// until the spawned pmux has returned, and returns a copy of Env with updated
// boostrap info.
func runDaemonPmuxOnce(env crypticnet.Env, s3Client garage.S3APIClient) (crypticnet.Env, error) {
func runDaemonPmuxOnce(env crypticnet.Env) (crypticnet.Env, error) {
thisHost := env.Bootstrap.ThisHost()
thisDaemon := env.ThisDaemon()
fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP)
// create s3Client anew on every loop, in case the topology has
// changed and we should be connecting to a different garage
// endpoint.
s3Client := env.Bootstrap.GlobalBucketS3APIClient()
nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(env)
if err != nil {
return crypticnet.Env{}, fmt.Errorf("generating nebula config: %w", err)
}
pmuxProcConfigs := []pmuxlib.ProcessConfig{
nebulaEntrypointPmuxProcConfig(),
nebulaPmuxProcConfig,
{
Name: "dnsmasq",
Cmd: "bash",
Args: waitForNebulaArgs(env, "dnsmasq-entrypoint"),
Args: []string{"dnsmasq-entrypoint"},
StartAfterFunc: func(ctx context.Context) error {
return waitForNebula(ctx, env)
},
},
}
if len(thisDaemon.Storage.Allocations) > 0 {
garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env)
garagePmuxProcConfigs, err := garagePmuxProcConfigs(env)
if err != nil {
return crypticnet.Env{}, fmt.Errorf("generating garage children configs: %w", err)
}
pmuxProcConfigs = append(pmuxProcConfigs, garageChildrenPmuxProcConfigs...)
pmuxProcConfigs = append(pmuxProcConfigs, garagePmuxProcConfigs...)
}
pmuxConfig := pmuxlib.Config{Processes: pmuxProcConfigs}
@ -123,14 +136,16 @@ func runDaemonPmuxOnce(env crypticnet.Env, s3Client garage.S3APIClient) (cryptic
go func() {
defer wg.Done()
// TODO wait for garage or nebula, depending on if allocs are present
if err := waitForGarageAndNebula(ctx, env); err != nil {
fmt.Fprintf(os.Stderr, "aborted waiting for garage instances to be accessible: %v\n", err)
return
}
client := env.Bootstrap.GlobalBucketS3APIClient()
thisHost := env.Bootstrap.ThisHost()
err := doOnce(ctx, func(ctx context.Context) error {
fmt.Fprintln(os.Stderr, "updating host info in garage")
return bootstrap.PutGarageBoostrapHost(ctx, client, thisHost)
return bootstrap.PutGarageBoostrapHost(ctx, s3Client, thisHost)
})
if err != nil {
@ -143,8 +158,8 @@ func runDaemonPmuxOnce(env crypticnet.Env, s3Client garage.S3APIClient) (cryptic
go func() {
defer wg.Done()
if err := waitForGarage(ctx, env); err != nil {
fmt.Fprintf(os.Stderr, "aborted waiting for garage instances to start: %v\n", err)
if err := waitForGarageAndNebula(ctx, env); err != nil {
fmt.Fprintf(os.Stderr, "aborted waiting for garage instances to be accessible: %v\n", err)
return
}
@ -287,6 +302,9 @@ var subCmdDaemon = subCmd{
return fmt.Errorf("merging daemon.yml into bootstrap data: %w", err)
}
// TODO once dnsmasq entrypoint is written in go and the config
// generation is inlined into this process then this Setenv won't be
// necessary.
for key, val := range env.ToMap() {
if err := os.Setenv(key, val); err != nil {
return fmt.Errorf("failed to set %q to %q: %w", key, val, err)
@ -295,12 +313,7 @@ var subCmdDaemon = subCmd{
for {
// create s3Client anew on every loop, in case the topology has
// changed and we should be connecting to a different garage
// endpoint.
s3Client := env.Bootstrap.GlobalBucketS3APIClient()
if env, err = runDaemonPmuxOnce(env, s3Client); errors.Is(err, context.Canceled) {
if env, err = runDaemonPmuxOnce(env); errors.Is(err, context.Canceled) {
return nil
} else if err != nil {

View File

@ -1,4 +1,4 @@
package entrypoint
package main
import (
"bytes"
@ -9,8 +9,7 @@ import (
"io"
"os"
"path/filepath"
"code.betamike.com/cryptic-io/pmux/pmuxlib"
"time"
)
func copyBootstrapToDataDirAndReload(env crypticnet.Env, r io.Reader) (crypticnet.Env, error) {
@ -74,20 +73,7 @@ func doOnce(ctx context.Context, fn func(context.Context) error) error {
} else if ctxErr := ctx.Err(); ctxErr != nil {
return ctxErr
}
}
}
func waitForNebulaArgs(env crypticnet.Env, args ...string) []string {
thisHost := env.Bootstrap.ThisHost()
return append([]string{"wait-for-ip", thisHost.Nebula.IP}, args...)
}
func nebulaEntrypointPmuxProcConfig() pmuxlib.ProcessConfig {
return pmuxlib.ProcessConfig{
Name: "nebula",
Cmd: "cryptic-net-main",
Args: []string{
"nebula-entrypoint",
},
time.Sleep(250 * time.Millisecond)
}
}

View File

@ -1,4 +1,4 @@
package entrypoint
package main
import (
crypticnet "cryptic-net"

View File

@ -1,4 +1,4 @@
package entrypoint
package main
import (
"fmt"

View File

@ -1,4 +1,4 @@
package entrypoint
package main
import (
"context"
@ -9,14 +9,21 @@ import (
"os"
"path/filepath"
"strconv"
"time"
"code.betamike.com/cryptic-io/pmux/pmuxlib"
)
func waitForGarage(ctx context.Context, env crypticnet.Env) error {
func waitForGarageAndNebula(ctx context.Context, env crypticnet.Env) error {
for _, alloc := range env.ThisDaemon().Storage.Allocations {
allocs := env.ThisDaemon().Storage.Allocations
// if this host doesn't have any allocations specified then fall back to
// waiting for nebula
if len(allocs) == 0 {
return waitForNebula(ctx, env)
}
for _, alloc := range allocs {
adminAddr := net.JoinHostPort(
env.Bootstrap.ThisHost().Nebula.IP,
@ -37,29 +44,6 @@ func waitForGarage(ctx context.Context, env crypticnet.Env) error {
}
func waitForGarageArgs(env crypticnet.Env, args ...string) []string {
thisHost := env.Bootstrap.ThisHost()
allocs := env.ThisDaemon().Storage.Allocations
if len(allocs) == 0 {
return waitForNebulaArgs(env, args...)
}
var preArgs []string
for _, alloc := range allocs {
preArgs = append(
preArgs,
"wait-for",
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
"--",
)
}
return append(preArgs, args...)
}
func garageWriteChildConf(
env crypticnet.Env,
alloc crypticnet.DaemonYmlStorageAllocation,
@ -116,7 +100,7 @@ func garageWriteChildConf(
return garageTomlPath, nil
}
func garageChildrenPmuxProcConfigs(env crypticnet.Env) ([]pmuxlib.ProcessConfig, error) {
func garagePmuxProcConfigs(env crypticnet.Env) ([]pmuxlib.ProcessConfig, error) {
var pmuxProcConfigs []pmuxlib.ProcessConfig
@ -129,25 +113,18 @@ func garageChildrenPmuxProcConfigs(env crypticnet.Env) ([]pmuxlib.ProcessConfig,
}
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
Name: fmt.Sprintf("garage-%d", alloc.RPCPort),
Cmd: "garage",
Args: []string{"-c", childConfPath, "server"},
SigKillWait: 1 * time.Minute,
Name: fmt.Sprintf("garage-%d", alloc.RPCPort),
Cmd: "garage",
Args: []string{"-c", childConfPath, "server"},
StartAfterFunc: func(ctx context.Context) error {
return waitForNebula(ctx, env)
},
})
}
return pmuxProcConfigs, nil
}
func garageApplyLayoutDiffPmuxProcConfig(env crypticnet.Env) pmuxlib.ProcessConfig {
return pmuxlib.ProcessConfig{
Name: "garage-apply-layout-diff",
Cmd: "bash",
Args: waitForGarageArgs(env, "bash", "garage-apply-layout-diff"),
NoRestartOn: []int{0},
}
}
func garageInitializeGlobalBucket(ctx context.Context, env crypticnet.Env) error {
var (

View File

@ -1,4 +1,4 @@
package entrypoint
package main
import (
"cryptic-net/bootstrap"

View File

@ -1,4 +1,4 @@
package entrypoint
package main
import (
"fmt"
@ -12,7 +12,7 @@ import (
// then passes execution along to an appropriate binary housed in AppDir/bin
// (usually a bash script, which is more versatile than a go program).
func Main() {
func main() {
env, err := crypticnet.NewEnv(true)

View File

@ -1,24 +1,40 @@
package nebula_entrypoint
package main
import (
"context"
crypticnet "cryptic-net"
"cryptic-net/yamlutil"
"log"
"fmt"
"net"
"os"
"path/filepath"
"strconv"
"syscall"
crypticnet "cryptic-net"
"code.betamike.com/cryptic-io/pmux/pmuxlib"
)
func Main() {
// waitForNebula waits for the nebula interface to have been started up. It does
// this by attempting to create a UDP connection which has the nebula IP set as
// its source. If this succeeds we can assume that at the very least the nebula
// interface has been initialized.
func waitForNebula(ctx context.Context, env crypticnet.Env) error {
env, err := crypticnet.ReadEnv()
ipStr := env.Bootstrap.ThisHost().Nebula.IP
ip := net.ParseIP(ipStr)
if err != nil {
log.Fatalf("reading envvars: %v", err)
}
lUdpAddr := &net.UDPAddr{IP: ip, Port: 0}
rUdpAddr := &net.UDPAddr{IP: ip, Port: 45535}
return doOnce(ctx, func(context.Context) error {
conn, err := net.DialUDP("udp", lUdpAddr, rUdpAddr)
if err != nil {
return err
}
conn.Close()
return nil
})
}
func nebulaPmuxProcConfig(env crypticnet.Env) (pmuxlib.ProcessConfig, error) {
var (
lighthouseHostIPs []string
@ -51,10 +67,6 @@ func Main() {
},
}
if err != nil {
log.Fatal(err)
}
if publicAddr := env.ThisDaemon().VPN.PublicAddr; publicAddr == "" {
config["listen"] = map[string]string{
@ -71,7 +83,7 @@ func Main() {
_, port, err := net.SplitHostPort(publicAddr)
if err != nil {
log.Fatalf("parsing public address %q: %v", publicAddr, err)
return pmuxlib.ProcessConfig{}, fmt.Errorf("parsing public address %q: %w", publicAddr, err)
}
config["listen"] = map[string]string{
@ -114,16 +126,12 @@ func Main() {
nebulaYmlPath := filepath.Join(env.RuntimeDirPath, "nebula.yml")
if err := yamlutil.WriteYamlFile(config, nebulaYmlPath); err != nil {
log.Fatalf("writing nebula.yml to %q: %v", nebulaYmlPath, err)
return pmuxlib.ProcessConfig{}, fmt.Errorf("writing nebula.yml to %q: %w", nebulaYmlPath, err)
}
var (
binPath = env.BinPath("nebula")
args = []string{"nebula", "-config", nebulaYmlPath}
cliEnv = os.Environ()
)
if err := syscall.Exec(binPath, args, cliEnv); err != nil {
log.Fatalf("calling exec(%q, %#v, %#v)", binPath, args, cliEnv)
}
return pmuxlib.ProcessConfig{
Name: "nebula",
Cmd: "nebula",
Args: []string{"-config", nebulaYmlPath},
}, nil
}

View File

@ -1,4 +1,4 @@
package entrypoint
package main
import (
crypticnet "cryptic-net"

View File

@ -1,4 +1,4 @@
package entrypoint
package main
import (
"fmt"

View File

@ -1,9 +0,0 @@
# go-workspace
This module is used for building all custom go binaries within the cryptic-net
project.
The reason binaries are contained here, and not under the sub-directory for the
sub-process the correspond to like most other code in this project, is that nix
makes it difficult to compose multiple modules defined locally. If nix ever
fixes this we should split this out.

View File

@ -1,18 +0,0 @@
{
buildGoModule,
}: let
build = subPackage: buildGoModule {
pname = "cryptic-net-" + (builtins.baseNameOf subPackage);
version = "unstable";
src = ./src;
vendorSha256 = "sha256-URmrK9Sd/5yhXrWxXZq05TS7aY7IWptQFMKfXKJY7Hc=";
subPackages = [
subPackage
];
};
in {
crypticNetMain = build "cmd/cryptic-net-main";
}

View File

@ -1,73 +0,0 @@
package main
//
// This binary acts as a wrapper around other programs which would otherwise
// form their own binaries. We do this for two reasons:
//
// * Nix makes it difficult to determine which individuals binaries need to be
// rebuilt upon changes, so it rebuilds all of them no matter what changed. This
// makes development slow. By wrapping everything in a sinble binary we only
// ever have to build that binary.
//
// * If we have N binaries total, then we have N copies of the go runtime in our
// final AppImage. By bundling the binaries into a single one we can reduce the
// number go runtime copies to 1.
//
import (
"cryptic-net/cmd/entrypoint"
nebula_entrypoint "cryptic-net/cmd/nebula-entrypoint"
"fmt"
"os"
)
type mainFn struct {
name string
fn func()
}
var mainFns = []mainFn{
{"entrypoint", entrypoint.Main},
{"nebula-entrypoint", nebula_entrypoint.Main},
}
var mainFnsMap = func() map[string]mainFn {
m := map[string]mainFn{}
for _, mainFn := range mainFns {
m[mainFn.name] = mainFn
}
return m
}()
func usage() {
fmt.Fprintf(os.Stderr, "USAGE: %s <cmd>\n\n", os.Args[0])
fmt.Fprintf(os.Stderr, "Commands:\n\n")
for _, mainFn := range mainFns {
fmt.Fprintf(os.Stderr, "%s\n", mainFn.name)
}
os.Stderr.Sync()
os.Exit(1)
}
func main() {
if len(os.Args) < 2 {
usage()
}
mainFn, ok := mainFnsMap[os.Args[1]]
if !ok {
usage()
}
// remove os.Args[1] from the arg list, so that other commands which consume
// args don't get confused
os.Args = append(os.Args[:1], os.Args[2:]...)
mainFn.fn()
}

View File

@ -16,6 +16,7 @@ rec {
in {
go = prev.go_1_18;
buildGoModule = args: prev.buildGoModule (buildArgs // args);
buildGo118Module = args: prev.buildGo118Module (buildArgs // args);
@ -31,11 +32,14 @@ rec {
];
stableSrc = fetchTarball {
name = "nixpkgs-22-05";
url = "https://github.com/NixOS/nixpkgs/archive/2aec372cdcd4d73b94863611fea70e0884270fdc.tar.gz";
version = "22-05";
rev = "2aec372cdcd4d73b94863611fea70e0884270fdc";
src = fetchTarball {
name = "nixpkgs-${version}";
url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz";
sha256 = "1pbfhlh4v8l70p44gspsci3i6w0wk70vaisiawg3jhka2nxb8367";
};
stable = import stableSrc { inherit overlays; };
pkgs = import src { inherit overlays; };
}

View File

@ -1,21 +0,0 @@
{
fetchFromGitHub,
stdenv,
}: stdenv.mkDerivation rec {
pname = "cryptic-net-wait-for";
version = "2.2.2";
src = fetchFromGitHub {
owner = "eficode";
repo = "wait-for";
rev = "v${version}";
sha256 = "sha256-qYeBOF63/+8bbFHiR6HT2mMQDFKCVkLNzIGLeEZJ4sk=";
};
builder = builtins.toFile "builder.sh" ''
source $stdenv/setup
mkdir -p "$out"/bin
cp "$src"/wait-for "$out"/bin/
'';
}