Compare commits

...

4 Commits

Author SHA1 Message Date
Brian Picciano
fb151865b4 WIP 2022-10-16 20:45:43 +02:00
Brian Picciano
0a826f86da Introduce admin.CreationParams 2022-10-16 20:45:42 +02:00
Brian Picciano
cf00561206 Factor out garage-entrypoint
The daemon entrypoint now starts the garage child processes directly,
without the extra step of indirection
2022-10-16 20:44:36 +02:00
Brian Picciano
18422a1084 Re-arrange sub-commands in entrypoint somewhat 2022-10-16 20:44:24 +02:00
14 changed files with 696 additions and 493 deletions

View File

@ -16,13 +16,15 @@ tmp="$(mktemp -d -t cryptic-net-dnsmasq-entrypoint-XXX)"
thisHostName=$(tar xzf "$_BOOTSTRAP_PATH" --to-stdout ./hostname)
thisHostIP=$(cat "$tmp"/hosts/"$thisHostName".yml | yq '.nebula.ip')
domain=$(tar xzf "$_BOOTSTRAP_PATH" --to-stdout ./admin/creation-params.yml | yq '.domain')
echo "listen-address=$thisHostIP" >> "$conf_path"
ls -1 "$tmp"/hosts | while read hostYml; do
hostName=$(echo "$hostYml" | cut -d. -f1)
hostIP=$(cat "$tmp"/hosts/"$hostYml" | yq '.nebula.ip')
echo "address=/${hostName}.hosts.cryptic.io/$hostIP" >> "$conf_path"
echo "address=/${hostName}.hosts.$domain/$hostIP" >> "$conf_path"
done
)

View File

@ -17,6 +17,7 @@ state AppDir {
entrypoint : * Merge given and default daemon.yml files
entrypoint : * Copy bootstrap.tgz into $_DATA_DIR_PATH, if it's not there
entrypoint : * Merge daemon.yml config into bootstrap.tgz
entrypoint : * Create $_RUNTIME_DIR_PATH/garage-N.toml\n (one per storage allocation)
entrypoint : * Run child processes
}
@ -41,20 +42,17 @@ state AppDir {
entrypoint --> nebulaEntrypoint : child
nebulaEntrypoint --> nebula : exec
state "./bin/cryptic-net-main garage-entrypoint" as garageEntrypoint {
garageEntrypoint : * Create $_RUNTIME_DIR_PATH/garage-N.toml\n (one per storage allocation)
garageEntrypoint : * Run child processes
}
state "Garage processes (only if any storage allocs are defined)" as garageChildren {
state "./bin/garage -c $_RUNTIME_DIR_PATH/garage-N.toml server" as garage
state "./bin/garage-apply-layout-diff" as garageApplyLayoutDiff {
garageApplyLayoutDiff : * Runs once then exits
garageApplyLayoutDiff : * Updates cluster topo
}
}
entrypoint --> garageEntrypoint : child (only if >1 storage allocation defined in daemon.yml)
garageEntrypoint --> garage : child (one per storage allocation)
garageEntrypoint --> garageApplyLayoutDiff : child
entrypoint --> garage : child (one per storage allocation)
entrypoint --> garageApplyLayoutDiff : child
state "./bin/cryptic-net-main update-global-bucket" as updateGlobalBucket {
updateGlobalBucket : * Runs once then exits

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 17 KiB

View File

@ -14,6 +14,8 @@ import (
)
const (
creationParamsPath = "admin/creation-params.yml"
nebulaCertsCACertPath = "nebula/certs/ca.crt"
nebulaCertsCAKeyPath = "nebula/certs/ca.key"
@ -22,8 +24,17 @@ const (
garageRPCSecretPath = "garage/rpc-secret.txt"
)
// CreationParams are general parameters used when creating a new network. These
// are available to all hosts within the network via their bootstrap files.
type CreationParams struct {
ID string `yaml:"id"`
Domain string `yaml:"domain"`
}
// Admin is used for accessing all information contained within an admin.tgz.
type Admin struct {
CreationParams CreationParams
NebulaCACert nebula.CACert
GarageRPCSecret string
@ -41,6 +52,7 @@ func FromFS(adminFS fs.FS) (Admin, error) {
into interface{}
path string
}{
{&a.CreationParams, creationParamsPath},
{&a.GarageGlobalBucketS3APICredentials, garageGlobalBucketKeyYmlPath},
{&a.GarageAdminBucketS3APICredentials, garageAdminBucketKeyYmlPath},
}
@ -91,6 +103,7 @@ func (a Admin) WriteTo(into io.Writer) error {
value interface{}
path string
}{
{a.CreationParams, creationParamsPath},
{a.GarageGlobalBucketS3APICredentials, garageGlobalBucketKeyYmlPath},
{a.GarageAdminBucketS3APICredentials, garageAdminBucketKeyYmlPath},
}

View File

@ -3,6 +3,7 @@
package bootstrap
import (
"cryptic-net/admin"
"cryptic-net/garage"
"cryptic-net/nebula"
"cryptic-net/tarutil"
@ -20,12 +21,15 @@ import (
// Paths within the bootstrap FS which for general data.
const (
adminCreationParamsPath = "admin/creation-params.yml"
hostNamePath = "hostname"
)
// Bootstrap is used for accessing all information contained within a
// bootstrap.tgz file.
type Bootstrap struct {
AdminCreationParams admin.CreationParams
Hosts map[string]Host
HostName string
@ -48,12 +52,18 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
return Bootstrap{}, fmt.Errorf("loading hosts info from fs: %w", err)
}
if err = yamlutil.LoadYamlFSFile(
&b.GarageGlobalBucketS3APICredentials,
bootstrapFS,
garageGlobalBucketKeyYmlPath,
); err != nil {
return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", garageGlobalBucketKeyYmlPath, err)
filesToLoadAsYAML := []struct {
into interface{}
path string
}{
{&b.AdminCreationParams, adminCreationParamsPath},
{&b.GarageGlobalBucketS3APICredentials, garageGlobalBucketKeyYmlPath},
}
for _, f := range filesToLoadAsYAML {
if err := yamlutil.LoadYamlFSFile(f.into, bootstrapFS, f.path); err != nil {
return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", f.path, err)
}
}
filesToLoadAsString := []struct {
@ -106,6 +116,36 @@ func (b Bootstrap) WriteTo(into io.Writer) error {
w := tarutil.NewTGZWriter(into)
for _, host := range b.Hosts {
hostB, err := yaml.Marshal(host)
if err != nil {
return fmt.Errorf("yaml encoding host %#v: %w", host, err)
}
path := filepath.Join(hostsDirPath, host.Name+".yml")
w.WriteFileBytes(path, hostB)
}
filesToWriteAsYAML := []struct {
value interface{}
path string
}{
{b.AdminCreationParams, adminCreationParamsPath},
{b.GarageGlobalBucketS3APICredentials, garageGlobalBucketKeyYmlPath},
}
for _, f := range filesToWriteAsYAML {
b, err := yaml.Marshal(f.value)
if err != nil {
return fmt.Errorf("yaml encoding data for %q: %w", f.path, err)
}
w.WriteFileBytes(f.path, b)
}
filesToWriteAsString := []struct {
value string
path string
@ -121,25 +161,6 @@ func (b Bootstrap) WriteTo(into io.Writer) error {
w.WriteFileBytes(f.path, []byte(f.value))
}
garageGlobalBucketKeyB, err := yaml.Marshal(b.GarageGlobalBucketS3APICredentials)
if err != nil {
return fmt.Errorf("yaml encoding garage global bucket creds: %w", err)
}
w.WriteFileBytes(garageGlobalBucketKeyYmlPath, garageGlobalBucketKeyB)
for _, host := range b.Hosts {
hostB, err := yaml.Marshal(host)
if err != nil {
return fmt.Errorf("yaml encoding host %#v: %w", host, err)
}
path := filepath.Join(hostsDirPath, host.Name+".yml")
w.WriteFileBytes(path, hostB)
}
return w.Close()
}

View File

@ -16,7 +16,6 @@ package main
import (
"cryptic-net/cmd/entrypoint"
garage_entrypoint "cryptic-net/cmd/garage-entrypoint"
garage_layout_diff "cryptic-net/cmd/garage-layout-diff"
garage_peer_keygen "cryptic-net/cmd/garage-peer-keygen"
nebula_entrypoint "cryptic-net/cmd/nebula-entrypoint"
@ -32,7 +31,6 @@ type mainFn struct {
var mainFns = []mainFn{
{"entrypoint", entrypoint.Main},
{"garage-entrypoint", garage_entrypoint.Main},
{"garage-layout-diff", garage_layout_diff.Main},
{"garage-peer-keygen", garage_peer_keygen.Main},
{"nebula-entrypoint", nebula_entrypoint.Main},

View File

@ -0,0 +1,278 @@
package entrypoint
import (
"context"
"cryptic-net/admin"
"cryptic-net/bootstrap"
"cryptic-net/nebula"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"net"
"os"
"strings"
"github.com/cryptic-io/pmux/pmuxlib"
)
func randStr(l int) string {
b := make([]byte, l)
if _, err := rand.Read(b); err != nil {
panic(err)
}
return hex.EncodeToString(b)
}
func readAdmin(path string) (admin.Admin, error) {
if path == "-" {
adm, err := admin.FromReader(os.Stdin)
if err != nil {
return admin.Admin{}, fmt.Errorf("parsing admin.tgz from stdin: %w", err)
}
return adm, nil
}
f, err := os.Open(path)
if err != nil {
return admin.Admin{}, fmt.Errorf("opening file: %w", err)
}
defer f.Close()
return admin.FromReader(f)
}
var subCmdAdminCreateNetwork = subCmd{
name: "create-network",
descr: "Creates a new cryptic-net network, outputting the resulting admin.tgz to stdout",
do: func(subCmdCtx subCmdCtx) error {
flags := subCmdCtx.flagSet(false)
daemonYmlPath := flags.StringP(
"config-path", "c", "",
"Optional path to a daemon.yml file to load configuration from.",
)
dumpConfig := flags.Bool(
"dump-config", false,
"Write the default configuration file to stdout and exit.",
)
domain := flags.StringP(
"domain", "d", "",
"Domain name that should be used as the root domain in the network.",
)
subnetStr := flags.StringP(
"subnet", "s", "",
"CIDR which denotes the subnet that IPs hosts on the network can be assigned.",
)
if err := flags.Parse(subCmdCtx.args); err != nil {
return fmt.Errorf("parsing flags: %w", err)
}
env := subCmdCtx.env
if *dumpConfig {
return writeBuiltinDaemonYml(env, os.Stdout)
}
if *domain == "" || *subnetStr == "" {
return errors.New("--domain and --subnet are required")
}
*domain = strings.TrimRight(strings.TrimLeft(*domain, "."), ".")
ip, subnet, err := net.ParseCIDR(*subnetStr)
if err != nil {
return fmt.Errorf("parsing %q as a CIDR: %w", *subnetStr, err)
}
hostName := "genesis"
adminCreationParams := admin.CreationParams{
ID: randStr(32),
Domain: *domain,
}
garageRPCSecret := randStr(32)
{
runtimeDirPath := env.RuntimeDirPath
fmt.Fprintf(os.Stderr, "will use runtime directory %q for temporary state\n", runtimeDirPath)
if err := os.MkdirAll(runtimeDirPath, 0700); err != nil {
return fmt.Errorf("creating directory %q: %w", runtimeDirPath, err)
}
defer func() {
fmt.Fprintf(os.Stderr, "cleaning up runtime directory %q\n", runtimeDirPath)
if err := os.RemoveAll(runtimeDirPath); err != nil {
fmt.Fprintf(os.Stderr, "error removing temporary directory %q: %v", runtimeDirPath, err)
}
}()
}
if err := writeMergedDaemonYml(env, *daemonYmlPath); err != nil {
return fmt.Errorf("merging and writing daemon.yml file: %w", err)
}
daemon := env.ThisDaemon()
if len(daemon.Storage.Allocations) < 3 {
return fmt.Errorf("daemon.yml with at least 3 allocations was not provided")
}
nebulaCACert, err := nebula.NewCACert(*domain, subnet)
if err != nil {
return fmt.Errorf("creating nebula CA cert: %w", err)
}
nebulaHostCert, err := nebula.NewHostCert(nebulaCACert, hostName, ip)
if err != nil {
return fmt.Errorf("creating nebula cert for host: %w", err)
}
host := bootstrap.Host{
Name: hostName,
Nebula: bootstrap.NebulaHost{
IP: ip.String(),
},
}
env.Bootstrap = bootstrap.Bootstrap{
AdminCreationParams: adminCreationParams,
Hosts: map[string]bootstrap.Host{
hostName: host,
},
HostName: hostName,
NebulaHostCert: nebulaHostCert,
GarageRPCSecret: garageRPCSecret,
}
// this will also write the bootstrap file
if err := mergeDaemonIntoBootstrap(env); err != nil {
return fmt.Errorf("merging daemon.yml into bootstrap data: %w", err)
}
for key, val := range env.ToMap() {
if err := os.Setenv(key, val); err != nil {
return fmt.Errorf("failed to set %q to %q: %w", key, val, err)
}
}
garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env)
if err != nil {
return fmt.Errorf("generating garage children configs: %w", err)
}
pmuxConfig := pmuxlib.Config{
Processes: append(
[]pmuxlib.ProcessConfig{
nebulaEntrypointPmuxProcConfig(),
garageApplyLayoutDiffPmuxProcConfig(env),
},
garageChildrenPmuxProcConfigs...,
),
}
ctx, cancel := context.WithCancel(env.Context)
defer cancel()
pmuxDoneCh := make(chan struct{})
go func() {
pmuxlib.Run(ctx, pmuxConfig)
close(pmuxDoneCh)
}()
},
}
var subCmdAdminMakeBootstrap = subCmd{
name: "make-bootstrap",
descr: "Creates a new bootstrap.tgz file for a particular host and writes it to stdout",
checkLock: true,
do: func(subCmdCtx subCmdCtx) error {
flags := subCmdCtx.flagSet(false)
name := flags.StringP(
"name", "n", "",
"Name of the host to generate bootstrap.tgz for",
)
adminPath := flags.StringP(
"admin-path", "a", "",
`Path to admin.tgz file. If the given path is "-" then stdin is used.`,
)
if err := flags.Parse(subCmdCtx.args); err != nil {
return fmt.Errorf("parsing flags: %w", err)
}
if *name == "" || *adminPath == "" {
return errors.New("--name and --admin-path are required")
}
env := subCmdCtx.env
adm, err := readAdmin(*adminPath)
if err != nil {
return fmt.Errorf("reading admin.tgz with --admin-path of %q: %w", *adminPath, err)
}
client, err := env.Bootstrap.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
// NOTE this isn't _technically_ required, but if the `hosts add`
// command for this host has been run recently then it might not have
// made it into the bootstrap file yet, and so won't be in
// `env.Bootstrap`.
hosts, err := bootstrap.GetGarageBootstrapHosts(env.Context, client)
if err != nil {
return fmt.Errorf("retrieving host info from garage: %w", err)
}
host, ok := hosts[*name]
if !ok {
return fmt.Errorf("couldn't find host into for %q in garage, has `cryptic-net hosts add` been run yet?", *name)
}
nebulaHostCert, err := nebula.NewHostCert(adm.NebulaCACert, host.Name, host.Nebula.IP)
if err != nil {
return fmt.Errorf("creating new nebula host key/cert: %w", err)
}
newBootstrap := bootstrap.Bootstrap{
AdminCreationParams: adm.CreationParams,
Hosts: hosts,
HostName: *name,
NebulaHostCert: nebulaHostCert,
GarageRPCSecret: adm.GarageRPCSecret,
GarageGlobalBucketS3APICredentials: adm.GarageGlobalBucketS3APICredentials,
}
return newBootstrap.WriteTo(os.Stdout)
},
}
var subCmdAdmin = subCmd{
name: "admin",
descr: "Sub-commands which only admins can run",
do: func(subCmdCtx subCmdCtx) error {
return subCmdCtx.doSubCmd(
subCmdAdminMakeBootstrap,
)
},
}

View File

@ -5,23 +5,15 @@ import (
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"net"
"os"
"path/filepath"
"strconv"
"sync"
"time"
crypticnet "cryptic-net"
"cryptic-net/bootstrap"
"cryptic-net/garage"
"cryptic-net/yamlutil"
"github.com/cryptic-io/pmux/pmuxlib"
"github.com/imdario/mergo"
"gopkg.in/yaml.v3"
)
// The daemon sub-command deals with starting an actual cryptic-net daemon
@ -46,70 +38,6 @@ import (
//
// * (On exit) cleans up the runtime directory.
func writeDaemonYml(userDaemonYmlPath, builtinDaemonYmlPath, runtimeDirPath string) error {
var fullDaemonYml map[string]interface{}
if err := yamlutil.LoadYamlFile(&fullDaemonYml, builtinDaemonYmlPath); err != nil {
return fmt.Errorf("parsing builtin daemon.yml file: %w", err)
}
if userDaemonYmlPath != "" {
var daemonYml map[string]interface{}
if err := yamlutil.LoadYamlFile(&daemonYml, userDaemonYmlPath); err != nil {
return fmt.Errorf("parsing %q: %w", userDaemonYmlPath, err)
}
err := mergo.Merge(&fullDaemonYml, daemonYml, mergo.WithOverride)
if err != nil {
return fmt.Errorf("merging contents of file %q: %w", userDaemonYmlPath, err)
}
}
fullDaemonYmlB, err := yaml.Marshal(fullDaemonYml)
if err != nil {
return fmt.Errorf("yaml marshaling daemon config: %w", err)
}
daemonYmlPath := filepath.Join(runtimeDirPath, "daemon.yml")
if err := ioutil.WriteFile(daemonYmlPath, fullDaemonYmlB, 0400); err != nil {
return fmt.Errorf("writing daemon.yml file to %q: %w", daemonYmlPath, err)
}
return nil
}
func copyBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
path := env.DataDirBootstrapPath()
dirPath := filepath.Dir(path)
if err := os.MkdirAll(dirPath, 0700); err != nil {
return fmt.Errorf("creating directory %q: %w", dirPath, err)
}
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("creating file %q: %w", path, err)
}
_, err = io.Copy(f, r)
f.Close()
if err != nil {
return fmt.Errorf("copying bootstrap file to %q: %w", path, err)
}
if err := env.LoadBootstrap(path); err != nil {
return fmt.Errorf("loading bootstrap from %q: %w", path, err)
}
return nil
}
// creates a new bootstrap file using available information from the network. If
// the new bootstrap file is different than the existing one, the existing one
// is overwritten, ReloadBootstrap is called on env, true is returned.
@ -156,65 +84,31 @@ func runDaemonPmuxOnce(env *crypticnet.Env, s3Client garage.S3APIClient) error {
fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP)
pmuxProcConfigs := []pmuxlib.ProcessConfig{
{
Name: "nebula",
Cmd: "cryptic-net-main",
Args: []string{
"nebula-entrypoint",
},
},
nebulaEntrypointPmuxProcConfig(),
{
Name: "dnsmasq",
Cmd: "bash",
Args: []string{
"wait-for-ip",
thisHost.Nebula.IP,
"dnsmasq-entrypoint",
},
Args: waitForNebulaArgs(env, "dnsmasq-entrypoint"),
},
}
{
var args []string
if len(thisDaemon.Storage.Allocations) > 0 {
if allocs := thisDaemon.Storage.Allocations; len(allocs) > 0 {
for _, alloc := range allocs {
args = append(
args,
"wait-for",
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
"--",
)
}
} else {
args = []string{
"wait-for-ip",
thisHost.Nebula.IP,
garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env)
if err != nil {
return fmt.Errorf("generating garage children configs: %w", err)
}
pmuxProcConfigs = append(pmuxProcConfigs, garageChildrenPmuxProcConfigs...)
pmuxProcConfigs = append(pmuxProcConfigs, garageApplyLayoutDiffPmuxProcConfig(env))
}
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
Name: "update-global-bucket",
Cmd: "bash",
Args: append(args, "update-global-bucket"),
Args: waitForGarageArgs(env, "update-global-bucket"),
NoRestartOn: []int{0},
})
}
if len(thisDaemon.Storage.Allocations) > 0 {
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
Name: "garage",
Cmd: "bash",
Args: []string{
"wait-for-ip",
thisHost.Nebula.IP,
"cryptic-net-main", "garage-entrypoint",
},
// garage can take a while to clean up
SigKillWait: (1 * time.Minute) + (10 * time.Second),
})
}
pmuxConfig := pmuxlib.Config{Processes: pmuxProcConfigs}
@ -284,28 +178,8 @@ var subCmdDaemon = subCmd{
env := subCmdCtx.env
s3Client, err := env.Bootstrap.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
appDirPath := env.AppDirPath
builtinDaemonYmlPath := filepath.Join(appDirPath, "etc", "daemon.yml")
if *dumpConfig {
builtinDaemonYml, err := os.ReadFile(builtinDaemonYmlPath)
if err != nil {
return fmt.Errorf("reading default daemon.yml at %q: %w", builtinDaemonYmlPath, err)
}
if _, err := os.Stdout.Write(builtinDaemonYml); err != nil {
return fmt.Errorf("writing default daemon.yml to stdout: %w", err)
}
return nil
return writeBuiltinDaemonYml(env, os.Stdout)
}
runtimeDirPath := env.RuntimeDirPath
@ -360,48 +234,17 @@ var subCmdDaemon = subCmd{
}
}
if err := writeDaemonYml(*daemonYmlPath, builtinDaemonYmlPath, runtimeDirPath); err != nil {
return fmt.Errorf("generating daemon.yml file: %w", err)
if err := writeMergedDaemonYml(env, *daemonYmlPath); err != nil {
return fmt.Errorf("merging and writing daemon.yml file: %w", err)
}
{
// we update this Host's data using whatever configuration has been
// provided by daemon.yml. This way the daemon has the most
// up-to-date possible bootstrap. This updated bootstrap will later
// get updated in garage using update-global-bucket, so other hosts
// will see it as well.
// ThisDaemon can only be called after writeDaemonYml.
daemon := env.ThisDaemon()
host := env.Bootstrap.ThisHost()
host.Nebula.PublicAddr = daemon.VPN.PublicAddr
host.Garage = nil
if allocs := daemon.Storage.Allocations; len(allocs) > 0 {
host.Garage = new(bootstrap.GarageHost)
for _, alloc := range allocs {
host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
WebPort: alloc.WebPort,
})
}
}
env.Bootstrap.Hosts[host.Name] = host
buf := new(bytes.Buffer)
if err := env.Bootstrap.WithHosts(env.Bootstrap.Hosts).WriteTo(buf); err != nil {
return fmt.Errorf("writing new bootstrap file to buffer: %w", err)
}
if err := copyBootstrapToDataDir(env, buf); err != nil {
return fmt.Errorf("copying new bootstrap file to data dir: %w", err)
}
if err := mergeDaemonIntoBootstrap(env); err != nil {
return fmt.Errorf("merging daemon.yml into bootstrap data: %w", err)
}
for key, val := range env.ToMap() {
@ -412,6 +255,14 @@ var subCmdDaemon = subCmd{
for {
// create s3Client anew on every loop, in case the topology has
// changed and we should be connecting to a different garage
// endpoint.
s3Client, err := env.Bootstrap.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
if err := runDaemonPmuxOnce(env, s3Client); errors.Is(err, context.Canceled) {
return nil

View File

@ -0,0 +1,205 @@
package entrypoint
import (
"bytes"
crypticnet "cryptic-net"
"cryptic-net/bootstrap"
"cryptic-net/garage"
"fmt"
"io"
"net"
"os"
"path/filepath"
"strconv"
"time"
"github.com/cryptic-io/pmux/pmuxlib"
)
func copyBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
path := env.DataDirBootstrapPath()
dirPath := filepath.Dir(path)
if err := os.MkdirAll(dirPath, 0700); err != nil {
return fmt.Errorf("creating directory %q: %w", dirPath, err)
}
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("creating file %q: %w", path, err)
}
_, err = io.Copy(f, r)
f.Close()
if err != nil {
return fmt.Errorf("copying bootstrap file to %q: %w", path, err)
}
if err := env.LoadBootstrap(path); err != nil {
return fmt.Errorf("loading bootstrap from %q: %w", path, err)
}
return nil
}
func mergeDaemonIntoBootstrap(env *crypticnet.Env) error {
daemon := env.ThisDaemon()
host := env.Bootstrap.ThisHost()
host.Nebula.PublicAddr = daemon.VPN.PublicAddr
host.Garage = nil
if allocs := daemon.Storage.Allocations; len(allocs) > 0 {
host.Garage = new(bootstrap.GarageHost)
for _, alloc := range allocs {
host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
WebPort: alloc.WebPort,
})
}
}
env.Bootstrap.Hosts[host.Name] = host
buf := new(bytes.Buffer)
if err := env.Bootstrap.WithHosts(env.Bootstrap.Hosts).WriteTo(buf); err != nil {
return fmt.Errorf("writing new bootstrap file to buffer: %w", err)
}
if err := copyBootstrapToDataDir(env, buf); err != nil {
return fmt.Errorf("copying new bootstrap file to data dir: %w", err)
}
return nil
}
func waitForNebulaArgs(env *crypticnet.Env, args ...string) []string {
thisHost := env.Bootstrap.ThisHost()
return append([]string{"wait-for-ip", thisHost.Nebula.IP}, args...)
}
func waitForGarageArgs(env *crypticnet.Env, args ...string) []string {
thisHost := env.Bootstrap.ThisHost()
allocs := env.ThisDaemon().Storage.Allocations
if len(allocs) == 0 {
return waitForNebulaArgs(env, args...)
}
var preArgs []string
for _, alloc := range allocs {
preArgs = append(
preArgs,
"wait-for",
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
"--",
)
}
return append(preArgs, args...)
}
func nebulaEntrypointPmuxProcConfig() pmuxlib.ProcessConfig {
return pmuxlib.ProcessConfig{
Name: "nebula",
Cmd: "cryptic-net-main",
Args: []string{
"nebula-entrypoint",
},
}
}
func garageWriteChildConf(
env *crypticnet.Env,
alloc crypticnet.DaemonYmlStorageAllocation,
) (
string, error,
) {
if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
}
thisHost := env.Bootstrap.ThisHost()
peer := garage.Peer{
IP: thisHost.Nebula.IP,
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
}
pubKey, privKey := peer.RPCPeerKey()
nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
}
garageTomlPath := filepath.Join(
env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
)
err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
MetaPath: alloc.MetaPath,
DataPath: alloc.DataPath,
RPCSecret: env.Bootstrap.GarageRPCSecret,
RPCAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
APIAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
WebAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.WebPort)),
BootstrapPeers: env.Bootstrap.GarageRPCPeerAddrs(),
})
if err != nil {
return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
}
return garageTomlPath, nil
}
func garageChildrenPmuxProcConfigs(env *crypticnet.Env) ([]pmuxlib.ProcessConfig, error) {
var pmuxProcConfigs []pmuxlib.ProcessConfig
for _, alloc := range env.ThisDaemon().Storage.Allocations {
childConfPath, err := garageWriteChildConf(env, alloc)
if err != nil {
return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
}
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
Name: fmt.Sprintf("garage-%d", alloc.RPCPort),
Cmd: "garage",
Args: []string{"-c", childConfPath, "server"},
SigKillWait: 1 * time.Minute,
})
}
return pmuxProcConfigs, nil
}
func garageApplyLayoutDiffPmuxProcConfig(env *crypticnet.Env) pmuxlib.ProcessConfig {
return pmuxlib.ProcessConfig{
Name: "garage-apply-layout-diff",
Cmd: "bash",
Args: waitForGarageArgs(env, "bash", "garage-apply-layout-diff"),
NoRestartOn: []int{0},
}
}

View File

@ -0,0 +1,72 @@
package entrypoint
import (
crypticnet "cryptic-net"
"cryptic-net/yamlutil"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"github.com/imdario/mergo"
"gopkg.in/yaml.v3"
)
func builtinDaemonYmlPath(env *crypticnet.Env) string {
return filepath.Join(env.AppDirPath, "etc", "daemon.yml")
}
func writeBuiltinDaemonYml(env *crypticnet.Env, w io.Writer) error {
builtinDaemonYmlPath := builtinDaemonYmlPath(env)
builtinDaemonYml, err := os.ReadFile(builtinDaemonYmlPath)
if err != nil {
return fmt.Errorf("reading default daemon.yml at %q: %w", builtinDaemonYmlPath, err)
}
if _, err := w.Write(builtinDaemonYml); err != nil {
return fmt.Errorf("writing default daemon.yml: %w", err)
}
return nil
}
func writeMergedDaemonYml(env *crypticnet.Env, userDaemonYmlPath string) error {
builtinDaemonYmlPath := builtinDaemonYmlPath(env)
var fullDaemonYml map[string]interface{}
if err := yamlutil.LoadYamlFile(&fullDaemonYml, builtinDaemonYmlPath); err != nil {
return fmt.Errorf("parsing builtin daemon.yml file: %w", err)
}
if userDaemonYmlPath != "" {
var daemonYml map[string]interface{}
if err := yamlutil.LoadYamlFile(&daemonYml, userDaemonYmlPath); err != nil {
return fmt.Errorf("parsing %q: %w", userDaemonYmlPath, err)
}
err := mergo.Merge(&fullDaemonYml, daemonYml, mergo.WithOverride)
if err != nil {
return fmt.Errorf("merging contents of file %q: %w", userDaemonYmlPath, err)
}
}
fullDaemonYmlB, err := yaml.Marshal(fullDaemonYml)
if err != nil {
return fmt.Errorf("yaml marshaling daemon config: %w", err)
}
daemonYmlPath := filepath.Join(env.RuntimeDirPath, "daemon.yml")
if err := ioutil.WriteFile(daemonYmlPath, fullDaemonYmlB, 0400); err != nil {
return fmt.Errorf("writing daemon.yml file to %q: %w", daemonYmlPath, err)
}
return nil
}

View File

@ -1,9 +1,7 @@
package entrypoint
import (
"cryptic-net/admin"
"cryptic-net/bootstrap"
"cryptic-net/nebula"
"errors"
"fmt"
"net"
@ -140,107 +138,14 @@ var subCmdHostsDelete = subCmd{
},
}
func readAdmin(path string) (admin.Admin, error) {
if path == "-" {
adm, err := admin.FromReader(os.Stdin)
if err != nil {
return admin.Admin{}, fmt.Errorf("parsing admin.tgz from stdin: %w", err)
}
return adm, nil
}
f, err := os.Open(path)
if err != nil {
return admin.Admin{}, fmt.Errorf("opening file: %w", err)
}
defer f.Close()
return admin.FromReader(f)
}
var subCmdHostsMakeBootstrap = subCmd{
name: "make-bootstrap",
descr: "Creates a new bootstrap.tgz file for a particular host and writes it to stdout",
checkLock: true,
do: func(subCmdCtx subCmdCtx) error {
flags := subCmdCtx.flagSet(false)
name := flags.StringP(
"name", "n", "",
"Name of the host to generate bootstrap.tgz for",
)
adminPath := flags.StringP(
"admin-path", "a", "",
`Path to admin.tgz file. If the given path is "-" then stdin is used.`,
)
if err := flags.Parse(subCmdCtx.args); err != nil {
return fmt.Errorf("parsing flags: %w", err)
}
if *name == "" || *adminPath == "" {
return errors.New("--name and --admin-path are required")
}
env := subCmdCtx.env
adm, err := readAdmin(*adminPath)
if err != nil {
return fmt.Errorf("reading admin.tgz with --admin-path of %q: %w", *adminPath, err)
}
client, err := env.Bootstrap.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
// NOTE this isn't _technically_ required, but if the `hosts add`
// command for this host has been run recently then it might not have
// made it into the bootstrap file yet, and so won't be in
// `env.Bootstrap`.
hosts, err := bootstrap.GetGarageBootstrapHosts(env.Context, client)
if err != nil {
return fmt.Errorf("retrieving host info from garage: %w", err)
}
host, ok := hosts[*name]
if !ok {
return fmt.Errorf("couldn't find host into for %q in garage, has `cryptic-net hosts add` been run yet?", *name)
}
nebulaHostCert, err := nebula.NewHostCert(adm.NebulaCACert, host.Name, host.Nebula.IP)
if err != nil {
return fmt.Errorf("creating new nebula host key/cert: %w", err)
}
newBootstrap := bootstrap.Bootstrap{
Hosts: hosts,
HostName: *name,
NebulaHostCert: nebulaHostCert,
GarageRPCSecret: adm.GarageRPCSecret,
GarageGlobalBucketS3APICredentials: adm.GarageGlobalBucketS3APICredentials,
}
return newBootstrap.WriteTo(os.Stdout)
},
}
var subCmdHosts = subCmd{
name: "hosts",
descr: "Sub-commands having to do with configuration of hosts in the network",
do: func(subCmdCtx subCmdCtx) error {
return subCmdCtx.doSubCmd(
subCmdHostsAdd,
subCmdHostsList,
subCmdHostsDelete,
subCmdHostsMakeBootstrap,
subCmdHostsList,
)
},
}

View File

@ -24,9 +24,10 @@ func Main() {
args: os.Args[1:],
env: env,
}.doSubCmd(
subCmdAdmin,
subCmdDaemon,
subCmdHosts,
subCmdGarage,
subCmdHosts,
subCmdVersion,
)

View File

@ -1,129 +0,0 @@
package garage_entrypoint
import (
"fmt"
"log"
"net"
"os"
"path/filepath"
"strconv"
"time"
crypticnet "cryptic-net"
"cryptic-net/garage"
"github.com/cryptic-io/pmux/pmuxlib"
)
func writeChildConf(
env *crypticnet.Env,
alloc crypticnet.DaemonYmlStorageAllocation,
) (string, error) {
if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
}
thisHost := env.Bootstrap.ThisHost()
peer := garage.Peer{
IP: thisHost.Nebula.IP,
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
}
pubKey, privKey := peer.RPCPeerKey()
nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
}
garageTomlPath := filepath.Join(
env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
)
err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
MetaPath: alloc.MetaPath,
DataPath: alloc.DataPath,
RPCSecret: env.Bootstrap.GarageRPCSecret,
RPCAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
APIAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
WebAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.WebPort)),
BootstrapPeers: env.Bootstrap.GarageRPCPeerAddrs(),
})
if err != nil {
return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
}
return garageTomlPath, nil
}
func waitForArgs(env *crypticnet.Env, bin string, binArgs ...string) []string {
thisHost := env.Bootstrap.ThisHost()
var args []string
for _, alloc := range env.ThisDaemon().Storage.Allocations {
args = append(
args,
"wait-for",
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
"--",
)
}
args = append(args, bin)
args = append(args, binArgs...)
return args
}
func Main() {
env, err := crypticnet.ReadEnv()
if err != nil {
log.Fatalf("reading envvars: %v", err)
}
var pmuxProcConfigs []pmuxlib.ProcessConfig
for _, alloc := range env.ThisDaemon().Storage.Allocations {
childConfPath, err := writeChildConf(env, alloc)
if err != nil {
log.Fatalf("writing child config file for alloc %+v: %v", alloc, err)
}
log.Printf("wrote config file %q", childConfPath)
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
Name: fmt.Sprintf("garage-%d", alloc.RPCPort),
Cmd: "garage",
Args: []string{"-c", childConfPath, "server"},
SigKillWait: 1 * time.Minute,
})
}
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
Name: "garage-apply-layout-diff",
Cmd: "bash",
Args: waitForArgs(env, "bash", "garage-apply-layout-diff"),
NoRestartOn: []int{0},
})
pmuxlib.Run(env.Context, pmuxlib.Config{Processes: pmuxProcConfigs})
}

View File

@ -14,15 +14,6 @@ import (
"golang.org/x/crypto/curve25519"
)
// TODO this should one day not be hardcoded
var ipCIDRMask = func() net.IPMask {
_, ipNet, err := net.ParseCIDR("10.10.0.0/16")
if err != nil {
panic(err)
}
return ipNet.Mask
}()
// HostCert contains the certificate and private key files which will need to
// be present on a particular host. Each file is PEM encoded.
type HostCert struct {
@ -41,7 +32,7 @@ type CACert struct {
// NewHostCert generates a new key/cert for a nebula host using the CA key
// which will be found in the adminFS.
func NewHostCert(
caCert CACert, hostName, hostIP string,
caCert CACert, hostName string, ip net.IP,
) (
HostCert, error,
) {
@ -66,14 +57,9 @@ func NewHostCert(
expireAt := caCrt.Details.NotAfter.Add(-1 * time.Second)
ip := net.ParseIP(hostIP)
if ip == nil {
return HostCert{}, fmt.Errorf("invalid host ip %q", hostIP)
}
ipNet := &net.IPNet{
IP: ip,
Mask: ipCIDRMask,
subnet := caCrt.Details.Subnets[0]
if !subnet.Contains(ip) {
return HostCert{}, fmt.Errorf("invalid ip %q, not contained by network subnet %q", ip, subnet)
}
var hostPub, hostKey []byte
@ -89,7 +75,10 @@ func NewHostCert(
hostCrt := cert.NebulaCertificate{
Details: cert.NebulaCertificateDetails{
Name: hostName,
Ips: []*net.IPNet{ipNet},
Ips: []*net.IPNet{{
IP: ip,
Mask: subnet.Mask,
}},
NotBefore: time.Now(),
NotAfter: expireAt,
PublicKey: hostPub,
@ -122,7 +111,7 @@ func NewHostCert(
// NewCACert generates a CACert. The domain should be the network's root domain,
// and is included in the signing certificate's Name field.
func NewCACert(domain string) (CACert, error) {
func NewCACert(domain string, subnet *net.IPNet) (CACert, error) {
pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
@ -135,6 +124,7 @@ func NewCACert(domain string) (CACert, error) {
caCrt := cert.NebulaCertificate{
Details: cert.NebulaCertificateDetails{
Name: fmt.Sprintf("%s cryptic-net root cert", domain),
Subnets: []*net.IPNet{subnet},
NotBefore: now,
NotAfter: expireAt,
PublicKey: pubKey,