Compare commits

..

No commits in common. "9288d8cf4880630f3428e93552b17b7f3fa9e644" and "8ba88b4dfca268bf156d7e2223ed3422cd25dd6a" have entirely different histories.

44 changed files with 262 additions and 147 deletions

View File

@ -1,4 +1,4 @@
#!/bin/sh
export PATH=$APPDIR/bin
exec entrypoint "$@"
exec cryptic-net-main entrypoint "$@"

9
AppDir/bin/wait-for-ip Normal file
View File

@ -0,0 +1,9 @@
ip="$1"
shift;
echo "waiting for $ip to become available..."
while true; do ping -c1 -W1 "$ip" &> /dev/null && break; done
exec "$@"

View File

@ -1,13 +1,9 @@
{
pkgsAttrs ? (import ./nix/pkgs.nix),
pkgs ? (import ./nix/pkgs.nix).stable,
bootstrap ? null,
}: let
pkgs = pkgsAttrs.pkgs;
in rec {
}: rec {
rootedBootstrap = pkgs.stdenv.mkDerivation {
name = "cryptic-net-rooted-bootstrap";
@ -25,37 +21,30 @@ in rec {
name = "cryptic-net-version";
buildInputs = [ pkgs.git pkgs.go ];
src = ./.;
inherit bootstrap;
nixPkgsVersion = pkgsAttrs.version;
nixPkgsRev = pkgsAttrs.rev;
builtByUser = builtins.getEnv "USER";
builder = builtins.toFile "builder.sh" ''
source $stdenv/setup
versionFile=version
cp -r "$src" srcCp
if [ "$bootstrap" != "" ]; then
hostName=$(tar -xzf "$bootstrap" --to-stdout ./hostname)
echo "Built for host: $hostName" >> "$versionFile"
fi
echo "Build date: $(date) ($(date +%s))" >> "$versionFile"
echo "Built by: $builtByUser" >> "$versionFile"
echo "Git rev: $(cd srcCp && git describe --always --long --dirty=' (dirty)')" >> "$versionFile"
echo "Build date: $(date)" >> "$versionFile"
echo "Git status: $(cd "$src" && git describe --always --long --dirty=' (dirty)')" >> "$versionFile"
echo "Go version: $(go version)" >> "$versionFile"
echo "Nixpkgs version: $nixPkgsVersion ($nixPkgsRev)" >> "$versionFile"
echo "Build host info: $(uname -srvm)" >> "$versionFile"
mkdir -p "$out"/share
cp "$versionFile" "$out"/share
'';
};
entrypoint = pkgs.callPackage ./entrypoint {};
goWorkspace = pkgs.callPackage ./go-workspace {};
dnsmasq = (pkgs.callPackage ./dnsmasq {
glibcStatic = pkgs.glibc.static;
@ -71,6 +60,8 @@ in rec {
pkgs.pkgsStatic.bash
pkgs.pkgsStatic.coreutils
pkgs.pkgsStatic.unixtools.ping
pkgs.pkgsStatic.netcat # required by waitFor
pkgs.pkgsStatic.gnutar
pkgs.pkgsStatic.gzip
@ -82,7 +73,8 @@ in rec {
version
dnsmasq
garage
entrypoint
waitFor
goWorkspace.crypticNetMain
] ++ (if bootstrap != null then [ rootedBootstrap ] else []);
};

View File

@ -1,6 +1,6 @@
{
pkgs ? (import ../nix/pkgs.nix).pkgs,
pkgs ? (import ../nix/pkgs.nix).stable,
}: pkgs.mkShell {
name = "cryptic-net-build-docs";

View File

@ -1,14 +0,0 @@
{
buildGoModule,
}: buildGoModule {
pname = "cryptic-net-entrypoint";
version = "unstable";
src = ./src;
vendorSha256 = "sha256-URmrK9Sd/5yhXrWxXZq05TS7aY7IWptQFMKfXKJY7Hc=";
subPackages = [
"cmd/entrypoint"
];
}

9
go-workspace/README.md Normal file
View File

@ -0,0 +1,9 @@
# go-workspace
This module is used for building all custom go binaries within the cryptic-net
project.
The reason binaries are contained here, and not under the sub-directory for the
sub-process the correspond to like most other code in this project, is that nix
makes it difficult to compose multiple modules defined locally. If nix ever
fixes this we should split this out.

18
go-workspace/default.nix Normal file
View File

@ -0,0 +1,18 @@
{
buildGoModule,
}: let
build = subPackage: buildGoModule {
pname = "cryptic-net-" + (builtins.baseNameOf subPackage);
version = "unstable";
src = ./src;
vendorSha256 = "sha256-URmrK9Sd/5yhXrWxXZq05TS7aY7IWptQFMKfXKJY7Hc=";
subPackages = [
subPackage
];
};
in {
crypticNetMain = build "cmd/cryptic-net-main";
}

View File

@ -0,0 +1,73 @@
package main
//
// This binary acts as a wrapper around other programs which would otherwise
// form their own binaries. We do this for two reasons:
//
// * Nix makes it difficult to determine which individuals binaries need to be
// rebuilt upon changes, so it rebuilds all of them no matter what changed. This
// makes development slow. By wrapping everything in a sinble binary we only
// ever have to build that binary.
//
// * If we have N binaries total, then we have N copies of the go runtime in our
// final AppImage. By bundling the binaries into a single one we can reduce the
// number go runtime copies to 1.
//
import (
"cryptic-net/cmd/entrypoint"
nebula_entrypoint "cryptic-net/cmd/nebula-entrypoint"
"fmt"
"os"
)
type mainFn struct {
name string
fn func()
}
var mainFns = []mainFn{
{"entrypoint", entrypoint.Main},
{"nebula-entrypoint", nebula_entrypoint.Main},
}
var mainFnsMap = func() map[string]mainFn {
m := map[string]mainFn{}
for _, mainFn := range mainFns {
m[mainFn.name] = mainFn
}
return m
}()
func usage() {
fmt.Fprintf(os.Stderr, "USAGE: %s <cmd>\n\n", os.Args[0])
fmt.Fprintf(os.Stderr, "Commands:\n\n")
for _, mainFn := range mainFns {
fmt.Fprintf(os.Stderr, "%s\n", mainFn.name)
}
os.Stderr.Sync()
os.Exit(1)
}
func main() {
if len(os.Args) < 2 {
usage()
}
mainFn, ok := mainFnsMap[os.Args[1]]
if !ok {
usage()
}
// remove os.Args[1] from the arg list, so that other commands which consume
// args don't get confused
os.Args = append(os.Args[:1], os.Args[2:]...)
mainFn.fn()
}

View File

@ -1,4 +1,4 @@
package main
package entrypoint
import (
"context"
@ -173,22 +173,17 @@ var subCmdAdminCreateNetwork = subCmd{
}
}
nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(env)
garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env)
if err != nil {
return fmt.Errorf("generating nebula config: %w", err)
}
garagePmuxProcConfigs, err := garagePmuxProcConfigs(env)
if err != nil {
return fmt.Errorf("generating garage configs: %w", err)
return fmt.Errorf("generating garage children configs: %w", err)
}
pmuxConfig := pmuxlib.Config{
Processes: append(
[]pmuxlib.ProcessConfig{
nebulaPmuxProcConfig,
nebulaEntrypointPmuxProcConfig(),
},
garagePmuxProcConfigs...,
garageChildrenPmuxProcConfigs...,
),
}
@ -208,7 +203,7 @@ var subCmdAdminCreateNetwork = subCmd{
}()
fmt.Fprintln(os.Stderr, "waiting for garage instances to come online")
if err := waitForGarageAndNebula(ctx, env); err != nil {
if err := waitForGarage(ctx, env); err != nil {
return fmt.Errorf("waiting for garage to start up: %w", err)
}

View File

@ -1,4 +1,4 @@
package main
package entrypoint
import (
"bytes"
@ -74,46 +74,33 @@ func reloadBootstrap(env crypticnet.Env, s3Client garage.S3APIClient) (crypticne
return env, true, nil
}
// runs a single pmux process of daemon, returning only once the env.Context has
// been canceled or bootstrap info has been changed. This will always block
// runs a single pmux process ofor daemon, returning only once the env.Context
// has been canceled or bootstrap info has been changed. This will always block
// until the spawned pmux has returned, and returns a copy of Env with updated
// boostrap info.
func runDaemonPmuxOnce(env crypticnet.Env) (crypticnet.Env, error) {
func runDaemonPmuxOnce(env crypticnet.Env, s3Client garage.S3APIClient) (crypticnet.Env, error) {
thisHost := env.Bootstrap.ThisHost()
thisDaemon := env.ThisDaemon()
fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP)
// create s3Client anew on every loop, in case the topology has
// changed and we should be connecting to a different garage
// endpoint.
s3Client := env.Bootstrap.GlobalBucketS3APIClient()
nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(env)
if err != nil {
return crypticnet.Env{}, fmt.Errorf("generating nebula config: %w", err)
}
pmuxProcConfigs := []pmuxlib.ProcessConfig{
nebulaPmuxProcConfig,
nebulaEntrypointPmuxProcConfig(),
{
Name: "dnsmasq",
Cmd: "bash",
Args: []string{"dnsmasq-entrypoint"},
StartAfterFunc: func(ctx context.Context) error {
return waitForNebula(ctx, env)
},
Args: waitForNebulaArgs(env, "dnsmasq-entrypoint"),
},
}
if len(thisDaemon.Storage.Allocations) > 0 {
garagePmuxProcConfigs, err := garagePmuxProcConfigs(env)
garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env)
if err != nil {
return crypticnet.Env{}, fmt.Errorf("generating garage children configs: %w", err)
}
pmuxProcConfigs = append(pmuxProcConfigs, garagePmuxProcConfigs...)
pmuxProcConfigs = append(pmuxProcConfigs, garageChildrenPmuxProcConfigs...)
}
pmuxConfig := pmuxlib.Config{Processes: pmuxProcConfigs}
@ -136,16 +123,14 @@ func runDaemonPmuxOnce(env crypticnet.Env) (crypticnet.Env, error) {
go func() {
defer wg.Done()
if err := waitForGarageAndNebula(ctx, env); err != nil {
fmt.Fprintf(os.Stderr, "aborted waiting for garage instances to be accessible: %v\n", err)
return
}
// TODO wait for garage or nebula, depending on if allocs are present
client := env.Bootstrap.GlobalBucketS3APIClient()
thisHost := env.Bootstrap.ThisHost()
err := doOnce(ctx, func(ctx context.Context) error {
fmt.Fprintln(os.Stderr, "updating host info in garage")
return bootstrap.PutGarageBoostrapHost(ctx, s3Client, thisHost)
return bootstrap.PutGarageBoostrapHost(ctx, client, thisHost)
})
if err != nil {
@ -158,8 +143,8 @@ func runDaemonPmuxOnce(env crypticnet.Env) (crypticnet.Env, error) {
go func() {
defer wg.Done()
if err := waitForGarageAndNebula(ctx, env); err != nil {
fmt.Fprintf(os.Stderr, "aborted waiting for garage instances to be accessible: %v\n", err)
if err := waitForGarage(ctx, env); err != nil {
fmt.Fprintf(os.Stderr, "aborted waiting for garage instances to start: %v\n", err)
return
}
@ -302,9 +287,6 @@ var subCmdDaemon = subCmd{
return fmt.Errorf("merging daemon.yml into bootstrap data: %w", err)
}
// TODO once dnsmasq entrypoint is written in go and the config
// generation is inlined into this process then this Setenv won't be
// necessary.
for key, val := range env.ToMap() {
if err := os.Setenv(key, val); err != nil {
return fmt.Errorf("failed to set %q to %q: %w", key, val, err)
@ -313,7 +295,12 @@ var subCmdDaemon = subCmd{
for {
if env, err = runDaemonPmuxOnce(env); errors.Is(err, context.Canceled) {
// create s3Client anew on every loop, in case the topology has
// changed and we should be connecting to a different garage
// endpoint.
s3Client := env.Bootstrap.GlobalBucketS3APIClient()
if env, err = runDaemonPmuxOnce(env, s3Client); errors.Is(err, context.Canceled) {
return nil
} else if err != nil {

View File

@ -1,4 +1,4 @@
package main
package entrypoint
import (
"bytes"
@ -9,7 +9,8 @@ import (
"io"
"os"
"path/filepath"
"time"
"code.betamike.com/cryptic-io/pmux/pmuxlib"
)
func copyBootstrapToDataDirAndReload(env crypticnet.Env, r io.Reader) (crypticnet.Env, error) {
@ -73,7 +74,20 @@ func doOnce(ctx context.Context, fn func(context.Context) error) error {
} else if ctxErr := ctx.Err(); ctxErr != nil {
return ctxErr
}
time.Sleep(250 * time.Millisecond)
}
}
func waitForNebulaArgs(env crypticnet.Env, args ...string) []string {
thisHost := env.Bootstrap.ThisHost()
return append([]string{"wait-for-ip", thisHost.Nebula.IP}, args...)
}
func nebulaEntrypointPmuxProcConfig() pmuxlib.ProcessConfig {
return pmuxlib.ProcessConfig{
Name: "nebula",
Cmd: "cryptic-net-main",
Args: []string{
"nebula-entrypoint",
},
}
}

View File

@ -1,4 +1,4 @@
package main
package entrypoint
import (
crypticnet "cryptic-net"

View File

@ -1,4 +1,4 @@
package main
package entrypoint
import (
"fmt"

View File

@ -1,4 +1,4 @@
package main
package entrypoint
import (
"context"
@ -9,21 +9,14 @@ import (
"os"
"path/filepath"
"strconv"
"time"
"code.betamike.com/cryptic-io/pmux/pmuxlib"
)
func waitForGarageAndNebula(ctx context.Context, env crypticnet.Env) error {
func waitForGarage(ctx context.Context, env crypticnet.Env) error {
allocs := env.ThisDaemon().Storage.Allocations
// if this host doesn't have any allocations specified then fall back to
// waiting for nebula
if len(allocs) == 0 {
return waitForNebula(ctx, env)
}
for _, alloc := range allocs {
for _, alloc := range env.ThisDaemon().Storage.Allocations {
adminAddr := net.JoinHostPort(
env.Bootstrap.ThisHost().Nebula.IP,
@ -44,6 +37,29 @@ func waitForGarageAndNebula(ctx context.Context, env crypticnet.Env) error {
}
func waitForGarageArgs(env crypticnet.Env, args ...string) []string {
thisHost := env.Bootstrap.ThisHost()
allocs := env.ThisDaemon().Storage.Allocations
if len(allocs) == 0 {
return waitForNebulaArgs(env, args...)
}
var preArgs []string
for _, alloc := range allocs {
preArgs = append(
preArgs,
"wait-for",
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
"--",
)
}
return append(preArgs, args...)
}
func garageWriteChildConf(
env crypticnet.Env,
alloc crypticnet.DaemonYmlStorageAllocation,
@ -100,7 +116,7 @@ func garageWriteChildConf(
return garageTomlPath, nil
}
func garagePmuxProcConfigs(env crypticnet.Env) ([]pmuxlib.ProcessConfig, error) {
func garageChildrenPmuxProcConfigs(env crypticnet.Env) ([]pmuxlib.ProcessConfig, error) {
var pmuxProcConfigs []pmuxlib.ProcessConfig
@ -113,18 +129,25 @@ func garagePmuxProcConfigs(env crypticnet.Env) ([]pmuxlib.ProcessConfig, error)
}
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
Name: fmt.Sprintf("garage-%d", alloc.RPCPort),
Cmd: "garage",
Args: []string{"-c", childConfPath, "server"},
StartAfterFunc: func(ctx context.Context) error {
return waitForNebula(ctx, env)
},
Name: fmt.Sprintf("garage-%d", alloc.RPCPort),
Cmd: "garage",
Args: []string{"-c", childConfPath, "server"},
SigKillWait: 1 * time.Minute,
})
}
return pmuxProcConfigs, nil
}
func garageApplyLayoutDiffPmuxProcConfig(env crypticnet.Env) pmuxlib.ProcessConfig {
return pmuxlib.ProcessConfig{
Name: "garage-apply-layout-diff",
Cmd: "bash",
Args: waitForGarageArgs(env, "bash", "garage-apply-layout-diff"),
NoRestartOn: []int{0},
}
}
func garageInitializeGlobalBucket(ctx context.Context, env crypticnet.Env) error {
var (

View File

@ -1,4 +1,4 @@
package main
package entrypoint
import (
"cryptic-net/bootstrap"

View File

@ -1,4 +1,4 @@
package main
package entrypoint
import (
"fmt"
@ -12,7 +12,7 @@ import (
// then passes execution along to an appropriate binary housed in AppDir/bin
// (usually a bash script, which is more versatile than a go program).
func main() {
func Main() {
env, err := crypticnet.NewEnv(true)

View File

@ -1,4 +1,4 @@
package main
package entrypoint
import (
crypticnet "cryptic-net"

View File

@ -1,4 +1,4 @@
package main
package entrypoint
import (
"fmt"

View File

@ -1,40 +1,24 @@
package main
package nebula_entrypoint
import (
"context"
crypticnet "cryptic-net"
"cryptic-net/yamlutil"
"fmt"
"log"
"net"
"os"
"path/filepath"
"strconv"
"syscall"
"code.betamike.com/cryptic-io/pmux/pmuxlib"
crypticnet "cryptic-net"
)
// waitForNebula waits for the nebula interface to have been started up. It does
// this by attempting to create a UDP connection which has the nebula IP set as
// its source. If this succeeds we can assume that at the very least the nebula
// interface has been initialized.
func waitForNebula(ctx context.Context, env crypticnet.Env) error {
func Main() {
ipStr := env.Bootstrap.ThisHost().Nebula.IP
ip := net.ParseIP(ipStr)
env, err := crypticnet.ReadEnv()
lUdpAddr := &net.UDPAddr{IP: ip, Port: 0}
rUdpAddr := &net.UDPAddr{IP: ip, Port: 45535}
return doOnce(ctx, func(context.Context) error {
conn, err := net.DialUDP("udp", lUdpAddr, rUdpAddr)
if err != nil {
return err
}
conn.Close()
return nil
})
}
func nebulaPmuxProcConfig(env crypticnet.Env) (pmuxlib.ProcessConfig, error) {
if err != nil {
log.Fatalf("reading envvars: %v", err)
}
var (
lighthouseHostIPs []string
@ -67,6 +51,10 @@ func nebulaPmuxProcConfig(env crypticnet.Env) (pmuxlib.ProcessConfig, error) {
},
}
if err != nil {
log.Fatal(err)
}
if publicAddr := env.ThisDaemon().VPN.PublicAddr; publicAddr == "" {
config["listen"] = map[string]string{
@ -83,7 +71,7 @@ func nebulaPmuxProcConfig(env crypticnet.Env) (pmuxlib.ProcessConfig, error) {
_, port, err := net.SplitHostPort(publicAddr)
if err != nil {
return pmuxlib.ProcessConfig{}, fmt.Errorf("parsing public address %q: %w", publicAddr, err)
log.Fatalf("parsing public address %q: %v", publicAddr, err)
}
config["listen"] = map[string]string{
@ -126,12 +114,16 @@ func nebulaPmuxProcConfig(env crypticnet.Env) (pmuxlib.ProcessConfig, error) {
nebulaYmlPath := filepath.Join(env.RuntimeDirPath, "nebula.yml")
if err := yamlutil.WriteYamlFile(config, nebulaYmlPath); err != nil {
return pmuxlib.ProcessConfig{}, fmt.Errorf("writing nebula.yml to %q: %w", nebulaYmlPath, err)
log.Fatalf("writing nebula.yml to %q: %v", nebulaYmlPath, err)
}
return pmuxlib.ProcessConfig{
Name: "nebula",
Cmd: "nebula",
Args: []string{"-config", nebulaYmlPath},
}, nil
var (
binPath = env.BinPath("nebula")
args = []string{"nebula", "-config", nebulaYmlPath}
cliEnv = os.Environ()
)
if err := syscall.Exec(binPath, args, cliEnv); err != nil {
log.Fatalf("calling exec(%q, %#v, %#v)", binPath, args, cliEnv)
}
}

View File

@ -16,7 +16,6 @@ rec {
in {
go = prev.go_1_18;
buildGoModule = args: prev.buildGoModule (buildArgs // args);
buildGo118Module = args: prev.buildGo118Module (buildArgs // args);
@ -32,14 +31,11 @@ rec {
];
version = "22-05";
rev = "2aec372cdcd4d73b94863611fea70e0884270fdc";
src = fetchTarball {
name = "nixpkgs-${version}";
url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz";
stableSrc = fetchTarball {
name = "nixpkgs-22-05";
url = "https://github.com/NixOS/nixpkgs/archive/2aec372cdcd4d73b94863611fea70e0884270fdc.tar.gz";
sha256 = "1pbfhlh4v8l70p44gspsci3i6w0wk70vaisiawg3jhka2nxb8367";
};
pkgs = import src { inherit overlays; };
stable = import stableSrc { inherit overlays; };
}

21
nix/wait-for.nix Normal file
View File

@ -0,0 +1,21 @@
{
fetchFromGitHub,
stdenv,
}: stdenv.mkDerivation rec {
pname = "cryptic-net-wait-for";
version = "2.2.2";
src = fetchFromGitHub {
owner = "eficode";
repo = "wait-for";
rev = "v${version}";
sha256 = "sha256-qYeBOF63/+8bbFHiR6HT2mMQDFKCVkLNzIGLeEZJ4sk=";
};
builder = builtins.toFile "builder.sh" ''
source $stdenv/setup
mkdir -p "$out"/bin
cp "$src"/wait-for "$out"/bin/
'';
}