Compare commits

...

6 Commits

Author SHA1 Message Date
Brian Picciano
c19b2f53dd WIP 2022-10-16 22:17:26 +02:00
Brian Picciano
7a25e1b6e6 Initial implementation of garage.AdminClient 2022-10-16 22:17:24 +02:00
Brian Picciano
eba9b23e61 Introduce admin.CreationParams 2022-10-16 22:07:03 +02:00
Brian Picciano
f720d7accd Enable the garage admin interface 2022-10-16 21:22:58 +02:00
Brian Picciano
51e21c3e46 Get rid of garage web port
It's not clear how we would be using it at this point, and garage 0.8.0
allows us to leave it off, so might as well do so.
2022-10-16 21:12:33 +02:00
Brian Picciano
5e08061cd6 Factor out garage-entrypoint
The daemon entrypoint now starts the garage child processes directly,
without the extra step of indirection
2022-10-16 20:48:33 +02:00
20 changed files with 776 additions and 420 deletions

View File

@ -64,7 +64,7 @@ storage:
# Capacity declares how many gigabytes can be stored in each allocation, and
# is required. It must be a multiple of 100.
#
# The various ports are all required and must all be unique within and across
# The ports are all required and must all be unique within and across
# allocations.
allocations:
@ -73,4 +73,4 @@ storage:
# capacity: 1200
# api_port: 3900
# rpc_port: 3901
# web_port: 3902
# admin_port: 3902

View File

@ -16,13 +16,15 @@ tmp="$(mktemp -d -t cryptic-net-dnsmasq-entrypoint-XXX)"
thisHostName=$(tar xzf "$_BOOTSTRAP_PATH" --to-stdout ./hostname)
thisHostIP=$(cat "$tmp"/hosts/"$thisHostName".yml | yq '.nebula.ip')
domain=$(tar xzf "$_BOOTSTRAP_PATH" --to-stdout ./admin/creation-params.yml | yq '.domain')
echo "listen-address=$thisHostIP" >> "$conf_path"
ls -1 "$tmp"/hosts | while read hostYml; do
hostName=$(echo "$hostYml" | cut -d. -f1)
hostIP=$(cat "$tmp"/hosts/"$hostYml" | yq '.nebula.ip')
echo "address=/${hostName}.hosts.cryptic.io/$hostIP" >> "$conf_path"
echo "address=/${hostName}.hosts.$domain/$hostIP" >> "$conf_path"
done
)

View File

@ -17,6 +17,7 @@ state AppDir {
entrypoint : * Merge given and default daemon.yml files
entrypoint : * Copy bootstrap.tgz into $_DATA_DIR_PATH, if it's not there
entrypoint : * Merge daemon.yml config into bootstrap.tgz
entrypoint : * Create $_RUNTIME_DIR_PATH/garage-N.toml\n (one per storage allocation)
entrypoint : * Run child processes
}
@ -41,20 +42,17 @@ state AppDir {
entrypoint --> nebulaEntrypoint : child
nebulaEntrypoint --> nebula : exec
state "./bin/cryptic-net-main garage-entrypoint" as garageEntrypoint {
garageEntrypoint : * Create $_RUNTIME_DIR_PATH/garage-N.toml\n (one per storage allocation)
garageEntrypoint : * Run child processes
}
state "Garage processes (only if any storage allocs are defined)" as garageChildren {
state "./bin/garage -c $_RUNTIME_DIR_PATH/garage-N.toml server" as garage
state "./bin/garage-apply-layout-diff" as garageApplyLayoutDiff {
garageApplyLayoutDiff : * Runs once then exits
garageApplyLayoutDiff : * Updates cluster topo
}
}
entrypoint --> garageEntrypoint : child (only if >1 storage allocation defined in daemon.yml)
garageEntrypoint --> garage : child (one per storage allocation)
garageEntrypoint --> garageApplyLayoutDiff : child
entrypoint --> garage : child (one per storage allocation)
entrypoint --> garageApplyLayoutDiff : child
state "./bin/cryptic-net-main update-global-bucket" as updateGlobalBucket {
updateGlobalBucket : * Runs once then exits

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 17 KiB

View File

@ -36,7 +36,6 @@ storage:
capacity: 1200
api_port: 3900
rpc_port: 3901
web_port: 3902
# 100 GB (the minimum) are being shared from drive2
- data_path: /mnt/drive2/cryptic-net/data
@ -44,7 +43,6 @@ storage:
capacity: 100
api_port: 3910
rpc_port: 3911
web_port: 3912
```
## Setup Firewall

View File

@ -14,6 +14,8 @@ import (
)
const (
creationParamsPath = "admin/creation-params.yml"
nebulaCertsCACertPath = "nebula/certs/ca.crt"
nebulaCertsCAKeyPath = "nebula/certs/ca.key"
@ -22,8 +24,17 @@ const (
garageRPCSecretPath = "garage/rpc-secret.txt"
)
// CreationParams are general parameters used when creating a new network. These
// are available to all hosts within the network via their bootstrap files.
type CreationParams struct {
ID string `yaml:"id"`
Domain string `yaml:"domain"`
}
// Admin is used for accessing all information contained within an admin.tgz.
type Admin struct {
CreationParams CreationParams
NebulaCACert nebula.CACert
GarageRPCSecret string
@ -41,6 +52,7 @@ func FromFS(adminFS fs.FS) (Admin, error) {
into interface{}
path string
}{
{&a.CreationParams, creationParamsPath},
{&a.GarageGlobalBucketS3APICredentials, garageGlobalBucketKeyYmlPath},
{&a.GarageAdminBucketS3APICredentials, garageAdminBucketKeyYmlPath},
}
@ -91,6 +103,7 @@ func (a Admin) WriteTo(into io.Writer) error {
value interface{}
path string
}{
{a.CreationParams, creationParamsPath},
{a.GarageGlobalBucketS3APICredentials, garageGlobalBucketKeyYmlPath},
{a.GarageAdminBucketS3APICredentials, garageAdminBucketKeyYmlPath},
}

View File

@ -3,6 +3,7 @@
package bootstrap
import (
"cryptic-net/admin"
"cryptic-net/garage"
"cryptic-net/nebula"
"cryptic-net/tarutil"
@ -20,18 +21,22 @@ import (
// Paths within the bootstrap FS which for general data.
const (
adminCreationParamsPath = "admin/creation-params.yml"
hostNamePath = "hostname"
)
// Bootstrap is used for accessing all information contained within a
// bootstrap.tgz file.
type Bootstrap struct {
AdminCreationParams admin.CreationParams
Hosts map[string]Host
HostName string
NebulaHostCert nebula.HostCert
GarageRPCSecret string
GarageAdminToken string
GarageGlobalBucketS3APICredentials garage.S3APICredentials
}
@ -48,12 +53,18 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
return Bootstrap{}, fmt.Errorf("loading hosts info from fs: %w", err)
}
if err = yamlutil.LoadYamlFSFile(
&b.GarageGlobalBucketS3APICredentials,
bootstrapFS,
garageGlobalBucketKeyYmlPath,
); err != nil {
return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", garageGlobalBucketKeyYmlPath, err)
filesToLoadAsYAML := []struct {
into interface{}
path string
}{
{&b.AdminCreationParams, adminCreationParamsPath},
{&b.GarageGlobalBucketS3APICredentials, garageGlobalBucketKeyYmlPath},
}
for _, f := range filesToLoadAsYAML {
if err := yamlutil.LoadYamlFSFile(f.into, bootstrapFS, f.path); err != nil {
return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", f.path, err)
}
}
filesToLoadAsString := []struct {
@ -65,6 +76,7 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
{&b.NebulaHostCert.HostCert, nebulaCertsHostCertPath},
{&b.NebulaHostCert.HostKey, nebulaCertsHostKeyPath},
{&b.GarageRPCSecret, garageRPCSecretPath},
{&b.GarageAdminToken, garageAdminTokenPath},
}
for _, f := range filesToLoadAsString {
@ -106,28 +118,6 @@ func (b Bootstrap) WriteTo(into io.Writer) error {
w := tarutil.NewTGZWriter(into)
filesToWriteAsString := []struct {
value string
path string
}{
{b.HostName, hostNamePath},
{b.NebulaHostCert.CACert, nebulaCertsCACertPath},
{b.NebulaHostCert.HostCert, nebulaCertsHostCertPath},
{b.NebulaHostCert.HostKey, nebulaCertsHostKeyPath},
{b.GarageRPCSecret, garageRPCSecretPath},
}
for _, f := range filesToWriteAsString {
w.WriteFileBytes(f.path, []byte(f.value))
}
garageGlobalBucketKeyB, err := yaml.Marshal(b.GarageGlobalBucketS3APICredentials)
if err != nil {
return fmt.Errorf("yaml encoding garage global bucket creds: %w", err)
}
w.WriteFileBytes(garageGlobalBucketKeyYmlPath, garageGlobalBucketKeyB)
for _, host := range b.Hosts {
hostB, err := yaml.Marshal(host)
@ -140,6 +130,40 @@ func (b Bootstrap) WriteTo(into io.Writer) error {
w.WriteFileBytes(path, hostB)
}
filesToWriteAsYAML := []struct {
value interface{}
path string
}{
{b.AdminCreationParams, adminCreationParamsPath},
{b.GarageGlobalBucketS3APICredentials, garageGlobalBucketKeyYmlPath},
}
for _, f := range filesToWriteAsYAML {
b, err := yaml.Marshal(f.value)
if err != nil {
return fmt.Errorf("yaml encoding data for %q: %w", f.path, err)
}
w.WriteFileBytes(f.path, b)
}
filesToWriteAsString := []struct {
value string
path string
}{
{b.HostName, hostNamePath},
{b.NebulaHostCert.CACert, nebulaCertsCACertPath},
{b.NebulaHostCert.HostCert, nebulaCertsHostCertPath},
{b.NebulaHostCert.HostKey, nebulaCertsHostKeyPath},
{b.GarageRPCSecret, garageRPCSecretPath},
{b.GarageAdminToken, garageAdminTokenPath},
}
for _, f := range filesToWriteAsString {
w.WriteFileBytes(f.path, []byte(f.value))
}
return w.Close()
}

View File

@ -7,8 +7,9 @@ import (
// Paths within the bootstrap FS related to garage.
const (
garageGlobalBucketKeyYmlPath = "garage/cryptic-net-global-bucket-key.yml"
garageRPCSecretPath = "garage/rpc-secret.txt"
garageAdminTokenPath = "garage/admin-token.txt"
garageGlobalBucketKeyYmlPath = "garage/cryptic-net-global-bucket-key.yml"
)
// GaragePeers returns a Peer for each known garage instance in the network.

View File

@ -24,7 +24,6 @@ type NebulaHost struct {
type GarageHostInstance struct {
RPCPort int `yaml:"rpc_port"`
S3APIPort int `yaml:"s3_api_port"`
WebPort int `yaml:"web_port"`
}
// GarageHost describes the garage configuration of a Host which is relevant for

View File

@ -16,7 +16,6 @@ package main
import (
"cryptic-net/cmd/entrypoint"
garage_entrypoint "cryptic-net/cmd/garage-entrypoint"
garage_layout_diff "cryptic-net/cmd/garage-layout-diff"
garage_peer_keygen "cryptic-net/cmd/garage-peer-keygen"
nebula_entrypoint "cryptic-net/cmd/nebula-entrypoint"
@ -32,7 +31,6 @@ type mainFn struct {
var mainFns = []mainFn{
{"entrypoint", entrypoint.Main},
{"garage-entrypoint", garage_entrypoint.Main},
{"garage-layout-diff", garage_layout_diff.Main},
{"garage-peer-keygen", garage_peer_keygen.Main},
{"nebula-entrypoint", nebula_entrypoint.Main},

View File

@ -1,14 +1,32 @@
package entrypoint
import (
"context"
crypticnet "cryptic-net"
"cryptic-net/admin"
"cryptic-net/bootstrap"
"cryptic-net/garage"
"cryptic-net/nebula"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"net"
"os"
"strconv"
"strings"
"github.com/cryptic-io/pmux/pmuxlib"
)
func randStr(l int) string {
b := make([]byte, l)
if _, err := rand.Read(b); err != nil {
panic(err)
}
return hex.EncodeToString(b)
}
func readAdmin(path string) (admin.Admin, error) {
if path == "-" {
@ -30,6 +48,236 @@ func readAdmin(path string) (admin.Admin, error) {
return admin.FromReader(f)
}
func garageInitializeGlobalBucket(
env *crypticnet.Env, globalBucketCreds garage.S3APICredentials,
) error {
var (
ctx = env.Context
thisHost = env.Bootstrap.ThisHost()
thisDaemon = env.ThisDaemon()
allocs = thisDaemon.Storage.Allocations
)
adminClient := garage.NewAdminClient(
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(allocs[0].AdminPort)),
env.Bootstrap.GarageAdminToken,
)
// first attempt to import the key
err := adminClient.Do(ctx, nil, "POST", "/v0/key/import", map[string]string{
"accessKeyId": globalBucketCreds.ID,
"secretAccessKey": globalBucketCreds.Secret,
"name": "shared-global-bucket-key",
})
if err != nil {
return fmt.Errorf("importing global bucket key into garage: %w", err)
}
// create global bucket
err = adminClient.Do(ctx, nil, "POST", "/v0/bucket", map[string]string{
"globalAlias": garage.GlobalBucket,
})
if err != nil {
return fmt.Errorf("creating global bucket: %w", err)
}
// retrieve newly created bucket's id
var getBucketRes struct {
ID string `json:"id"`
}
err = adminClient.Do(
ctx, &getBucketRes,
"GET", "/v0/bucket?globalAlias="+garage.GlobalBucket, nil,
)
if err != nil {
return fmt.Errorf("fetching global bucket id: %w", err)
}
// allow shared global bucket key to perform all operations
err = adminClient.Do(ctx, nil, "POST", "/v0/bucket/allow", map[string]interface{}{
"bucketId": getBucketRes.ID,
"accessKeyId": globalBucketCreds.ID,
"permissions": map[string]bool{
"read": true,
"write": true,
},
})
if err != nil {
return fmt.Errorf("granting permissions to shared global bucket key: %w", err)
}
return nil
}
var subCmdAdminCreateNetwork = subCmd{
name: "create-network",
descr: "Creates a new cryptic-net network, outputting the resulting admin.tgz to stdout",
do: func(subCmdCtx subCmdCtx) error {
flags := subCmdCtx.flagSet(false)
daemonYmlPath := flags.StringP(
"config-path", "c", "",
"Optional path to a daemon.yml file to load configuration from.",
)
dumpConfig := flags.Bool(
"dump-config", false,
"Write the default configuration file to stdout and exit.",
)
domain := flags.StringP(
"domain", "d", "",
"Domain name that should be used as the root domain in the network.",
)
subnetStr := flags.StringP(
"subnet", "s", "",
"CIDR which denotes the subnet that IPs hosts on the network can be assigned.",
)
if err := flags.Parse(subCmdCtx.args); err != nil {
return fmt.Errorf("parsing flags: %w", err)
}
env := subCmdCtx.env
if *dumpConfig {
return writeBuiltinDaemonYml(env, os.Stdout)
}
if *domain == "" || *subnetStr == "" {
return errors.New("--domain and --subnet are required")
}
*domain = strings.TrimRight(strings.TrimLeft(*domain, "."), ".")
ip, subnet, err := net.ParseCIDR(*subnetStr)
if err != nil {
return fmt.Errorf("parsing %q as a CIDR: %w", *subnetStr, err)
}
hostName := "genesis"
adminCreationParams := admin.CreationParams{
ID: randStr(32),
Domain: *domain,
}
garageRPCSecret := randStr(32)
{
runtimeDirPath := env.RuntimeDirPath
fmt.Fprintf(os.Stderr, "will use runtime directory %q for temporary state\n", runtimeDirPath)
if err := os.MkdirAll(runtimeDirPath, 0700); err != nil {
return fmt.Errorf("creating directory %q: %w", runtimeDirPath, err)
}
defer func() {
fmt.Fprintf(os.Stderr, "cleaning up runtime directory %q\n", runtimeDirPath)
if err := os.RemoveAll(runtimeDirPath); err != nil {
fmt.Fprintf(os.Stderr, "error removing temporary directory %q: %v", runtimeDirPath, err)
}
}()
}
if err := writeMergedDaemonYml(env, *daemonYmlPath); err != nil {
return fmt.Errorf("merging and writing daemon.yml file: %w", err)
}
daemon := env.ThisDaemon()
if len(daemon.Storage.Allocations) < 3 {
return fmt.Errorf("daemon.yml with at least 3 allocations was not provided")
}
nebulaCACert, err := nebula.NewCACert(*domain, subnet)
if err != nil {
return fmt.Errorf("creating nebula CA cert: %w", err)
}
nebulaHostCert, err := nebula.NewHostCert(nebulaCACert, hostName, ip)
if err != nil {
return fmt.Errorf("creating nebula cert for host: %w", err)
}
host := bootstrap.Host{
Name: hostName,
Nebula: bootstrap.NebulaHost{
IP: ip.String(),
},
}
env.Bootstrap = bootstrap.Bootstrap{
AdminCreationParams: adminCreationParams,
Hosts: map[string]bootstrap.Host{
hostName: host,
},
HostName: hostName,
NebulaHostCert: nebulaHostCert,
GarageRPCSecret: garageRPCSecret,
}
// this will also write the bootstrap file
if err := mergeDaemonIntoBootstrap(env); err != nil {
return fmt.Errorf("merging daemon.yml into bootstrap data: %w", err)
}
for key, val := range env.ToMap() {
if err := os.Setenv(key, val); err != nil {
return fmt.Errorf("failed to set %q to %q: %w", key, val, err)
}
}
garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env)
if err != nil {
return fmt.Errorf("generating garage children configs: %w", err)
}
pmuxConfig := pmuxlib.Config{
Processes: append(
[]pmuxlib.ProcessConfig{
nebulaEntrypointPmuxProcConfig(),
garageApplyLayoutDiffPmuxProcConfig(env),
},
garageChildrenPmuxProcConfigs...,
),
}
ctx, cancel := context.WithCancel(env.Context)
pmuxDoneCh := make(chan struct{})
go func() {
pmuxlib.Run(ctx, pmuxConfig)
close(pmuxDoneCh)
}()
defer func() {
cancel()
<-pmuxDoneCh
}()
globalBucketCreds := garage.S3APICredentials{} // TODO
// TODO wait for garage to be confirmed as booted up
// TODO apply layout
if err := garageInitializeGlobalBucket(env, globalBucketCreds); err != nil {
return fmt.Errorf("initializing shared global bucket: %w", err)
}
panic("TODO: create and output admin.tgz")
},
}
var subCmdAdminMakeBootstrap = subCmd{
name: "make-bootstrap",
descr: "Creates a new bootstrap.tgz file for a particular host and writes it to stdout",
@ -82,18 +330,26 @@ var subCmdAdminMakeBootstrap = subCmd{
return fmt.Errorf("couldn't find host into for %q in garage, has `cryptic-net hosts add` been run yet?", *name)
}
nebulaHostCert, err := nebula.NewHostCert(adm.NebulaCACert, host.Name, host.Nebula.IP)
ip := net.ParseIP(host.Nebula.IP)
if ip == nil {
return fmt.Errorf("invalid IP stored with host %q: %q", *name, host.Nebula.IP)
}
nebulaHostCert, err := nebula.NewHostCert(adm.NebulaCACert, host.Name, ip)
if err != nil {
return fmt.Errorf("creating new nebula host key/cert: %w", err)
}
newBootstrap := bootstrap.Bootstrap{
AdminCreationParams: adm.CreationParams,
Hosts: hosts,
HostName: *name,
NebulaHostCert: nebulaHostCert,
GarageRPCSecret: adm.GarageRPCSecret,
GarageAdminToken: randStr(32),
GarageGlobalBucketS3APICredentials: adm.GarageGlobalBucketS3APICredentials,
}

View File

@ -5,23 +5,15 @@ import (
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"net"
"os"
"path/filepath"
"strconv"
"sync"
"time"
crypticnet "cryptic-net"
"cryptic-net/bootstrap"
"cryptic-net/garage"
"cryptic-net/yamlutil"
"github.com/cryptic-io/pmux/pmuxlib"
"github.com/imdario/mergo"
"gopkg.in/yaml.v3"
)
// The daemon sub-command deals with starting an actual cryptic-net daemon
@ -46,70 +38,6 @@ import (
//
// * (On exit) cleans up the runtime directory.
func writeDaemonYml(userDaemonYmlPath, builtinDaemonYmlPath, runtimeDirPath string) error {
var fullDaemonYml map[string]interface{}
if err := yamlutil.LoadYamlFile(&fullDaemonYml, builtinDaemonYmlPath); err != nil {
return fmt.Errorf("parsing builtin daemon.yml file: %w", err)
}
if userDaemonYmlPath != "" {
var daemonYml map[string]interface{}
if err := yamlutil.LoadYamlFile(&daemonYml, userDaemonYmlPath); err != nil {
return fmt.Errorf("parsing %q: %w", userDaemonYmlPath, err)
}
err := mergo.Merge(&fullDaemonYml, daemonYml, mergo.WithOverride)
if err != nil {
return fmt.Errorf("merging contents of file %q: %w", userDaemonYmlPath, err)
}
}
fullDaemonYmlB, err := yaml.Marshal(fullDaemonYml)
if err != nil {
return fmt.Errorf("yaml marshaling daemon config: %w", err)
}
daemonYmlPath := filepath.Join(runtimeDirPath, "daemon.yml")
if err := ioutil.WriteFile(daemonYmlPath, fullDaemonYmlB, 0400); err != nil {
return fmt.Errorf("writing daemon.yml file to %q: %w", daemonYmlPath, err)
}
return nil
}
func copyBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
path := env.DataDirBootstrapPath()
dirPath := filepath.Dir(path)
if err := os.MkdirAll(dirPath, 0700); err != nil {
return fmt.Errorf("creating directory %q: %w", dirPath, err)
}
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("creating file %q: %w", path, err)
}
_, err = io.Copy(f, r)
f.Close()
if err != nil {
return fmt.Errorf("copying bootstrap file to %q: %w", path, err)
}
if err := env.LoadBootstrap(path); err != nil {
return fmt.Errorf("loading bootstrap from %q: %w", path, err)
}
return nil
}
// creates a new bootstrap file using available information from the network. If
// the new bootstrap file is different than the existing one, the existing one
// is overwritten, ReloadBootstrap is called on env, true is returned.
@ -156,65 +84,31 @@ func runDaemonPmuxOnce(env *crypticnet.Env, s3Client garage.S3APIClient) error {
fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP)
pmuxProcConfigs := []pmuxlib.ProcessConfig{
{
Name: "nebula",
Cmd: "cryptic-net-main",
Args: []string{
"nebula-entrypoint",
},
},
nebulaEntrypointPmuxProcConfig(),
{
Name: "dnsmasq",
Cmd: "bash",
Args: []string{
"wait-for-ip",
thisHost.Nebula.IP,
"dnsmasq-entrypoint",
},
Args: waitForNebulaArgs(env, "dnsmasq-entrypoint"),
},
}
{
var args []string
if len(thisDaemon.Storage.Allocations) > 0 {
if allocs := thisDaemon.Storage.Allocations; len(allocs) > 0 {
for _, alloc := range allocs {
args = append(
args,
"wait-for",
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
"--",
)
}
} else {
args = []string{
"wait-for-ip",
thisHost.Nebula.IP,
garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env)
if err != nil {
return fmt.Errorf("generating garage children configs: %w", err)
}
pmuxProcConfigs = append(pmuxProcConfigs, garageChildrenPmuxProcConfigs...)
pmuxProcConfigs = append(pmuxProcConfigs, garageApplyLayoutDiffPmuxProcConfig(env))
}
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
Name: "update-global-bucket",
Cmd: "bash",
Args: append(args, "update-global-bucket"),
Args: waitForGarageArgs(env, "update-global-bucket"),
NoRestartOn: []int{0},
})
}
if len(thisDaemon.Storage.Allocations) > 0 {
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
Name: "garage",
Cmd: "bash",
Args: []string{
"wait-for-ip",
thisHost.Nebula.IP,
"cryptic-net-main", "garage-entrypoint",
},
// garage can take a while to clean up
SigKillWait: (1 * time.Minute) + (10 * time.Second),
})
}
pmuxConfig := pmuxlib.Config{Processes: pmuxProcConfigs}
@ -284,28 +178,8 @@ var subCmdDaemon = subCmd{
env := subCmdCtx.env
s3Client, err := env.Bootstrap.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
appDirPath := env.AppDirPath
builtinDaemonYmlPath := filepath.Join(appDirPath, "etc", "daemon.yml")
if *dumpConfig {
builtinDaemonYml, err := os.ReadFile(builtinDaemonYmlPath)
if err != nil {
return fmt.Errorf("reading default daemon.yml at %q: %w", builtinDaemonYmlPath, err)
}
if _, err := os.Stdout.Write(builtinDaemonYml); err != nil {
return fmt.Errorf("writing default daemon.yml to stdout: %w", err)
}
return nil
return writeBuiltinDaemonYml(env, os.Stdout)
}
runtimeDirPath := env.RuntimeDirPath
@ -360,48 +234,17 @@ var subCmdDaemon = subCmd{
}
}
if err := writeDaemonYml(*daemonYmlPath, builtinDaemonYmlPath, runtimeDirPath); err != nil {
return fmt.Errorf("generating daemon.yml file: %w", err)
if err := writeMergedDaemonYml(env, *daemonYmlPath); err != nil {
return fmt.Errorf("merging and writing daemon.yml file: %w", err)
}
{
// we update this Host's data using whatever configuration has been
// provided by daemon.yml. This way the daemon has the most
// up-to-date possible bootstrap. This updated bootstrap will later
// get updated in garage using update-global-bucket, so other hosts
// will see it as well.
// ThisDaemon can only be called after writeDaemonYml.
daemon := env.ThisDaemon()
host := env.Bootstrap.ThisHost()
host.Nebula.PublicAddr = daemon.VPN.PublicAddr
host.Garage = nil
if allocs := daemon.Storage.Allocations; len(allocs) > 0 {
host.Garage = new(bootstrap.GarageHost)
for _, alloc := range allocs {
host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
WebPort: alloc.WebPort,
})
}
}
env.Bootstrap.Hosts[host.Name] = host
buf := new(bytes.Buffer)
if err := env.Bootstrap.WithHosts(env.Bootstrap.Hosts).WriteTo(buf); err != nil {
return fmt.Errorf("writing new bootstrap file to buffer: %w", err)
}
if err := copyBootstrapToDataDir(env, buf); err != nil {
return fmt.Errorf("copying new bootstrap file to data dir: %w", err)
}
if err := mergeDaemonIntoBootstrap(env); err != nil {
return fmt.Errorf("merging daemon.yml into bootstrap data: %w", err)
}
for key, val := range env.ToMap() {
@ -412,6 +255,14 @@ var subCmdDaemon = subCmd{
for {
// create s3Client anew on every loop, in case the topology has
// changed and we should be connecting to a different garage
// endpoint.
s3Client, err := env.Bootstrap.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
if err := runDaemonPmuxOnce(env, s3Client); errors.Is(err, context.Canceled) {
return nil

View File

@ -0,0 +1,205 @@
package entrypoint
import (
"bytes"
crypticnet "cryptic-net"
"cryptic-net/bootstrap"
"cryptic-net/garage"
"fmt"
"io"
"net"
"os"
"path/filepath"
"strconv"
"time"
"github.com/cryptic-io/pmux/pmuxlib"
)
func copyBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
path := env.DataDirBootstrapPath()
dirPath := filepath.Dir(path)
if err := os.MkdirAll(dirPath, 0700); err != nil {
return fmt.Errorf("creating directory %q: %w", dirPath, err)
}
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("creating file %q: %w", path, err)
}
_, err = io.Copy(f, r)
f.Close()
if err != nil {
return fmt.Errorf("copying bootstrap file to %q: %w", path, err)
}
if err := env.LoadBootstrap(path); err != nil {
return fmt.Errorf("loading bootstrap from %q: %w", path, err)
}
return nil
}
func mergeDaemonIntoBootstrap(env *crypticnet.Env) error {
daemon := env.ThisDaemon()
host := env.Bootstrap.ThisHost()
host.Nebula.PublicAddr = daemon.VPN.PublicAddr
host.Garage = nil
if allocs := daemon.Storage.Allocations; len(allocs) > 0 {
host.Garage = new(bootstrap.GarageHost)
for _, alloc := range allocs {
host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
})
}
}
env.Bootstrap.Hosts[host.Name] = host
buf := new(bytes.Buffer)
if err := env.Bootstrap.WithHosts(env.Bootstrap.Hosts).WriteTo(buf); err != nil {
return fmt.Errorf("writing new bootstrap file to buffer: %w", err)
}
if err := copyBootstrapToDataDir(env, buf); err != nil {
return fmt.Errorf("copying new bootstrap file to data dir: %w", err)
}
return nil
}
func waitForNebulaArgs(env *crypticnet.Env, args ...string) []string {
thisHost := env.Bootstrap.ThisHost()
return append([]string{"wait-for-ip", thisHost.Nebula.IP}, args...)
}
func waitForGarageArgs(env *crypticnet.Env, args ...string) []string {
thisHost := env.Bootstrap.ThisHost()
allocs := env.ThisDaemon().Storage.Allocations
if len(allocs) == 0 {
return waitForNebulaArgs(env, args...)
}
var preArgs []string
for _, alloc := range allocs {
preArgs = append(
preArgs,
"wait-for",
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
"--",
)
}
return append(preArgs, args...)
}
func nebulaEntrypointPmuxProcConfig() pmuxlib.ProcessConfig {
return pmuxlib.ProcessConfig{
Name: "nebula",
Cmd: "cryptic-net-main",
Args: []string{
"nebula-entrypoint",
},
}
}
func garageWriteChildConf(
env *crypticnet.Env,
alloc crypticnet.DaemonYmlStorageAllocation,
) (
string, error,
) {
if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
}
thisHost := env.Bootstrap.ThisHost()
peer := garage.Peer{
IP: thisHost.Nebula.IP,
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
}
pubKey, privKey := peer.RPCPeerKey()
nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
}
garageTomlPath := filepath.Join(
env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
)
err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
MetaPath: alloc.MetaPath,
DataPath: alloc.DataPath,
RPCSecret: env.Bootstrap.GarageRPCSecret,
AdminToken: env.Bootstrap.GarageAdminToken,
RPCAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
APIAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
AdminAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.AdminPort)),
BootstrapPeers: env.Bootstrap.GarageRPCPeerAddrs(),
})
if err != nil {
return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
}
return garageTomlPath, nil
}
func garageChildrenPmuxProcConfigs(env *crypticnet.Env) ([]pmuxlib.ProcessConfig, error) {
var pmuxProcConfigs []pmuxlib.ProcessConfig
for _, alloc := range env.ThisDaemon().Storage.Allocations {
childConfPath, err := garageWriteChildConf(env, alloc)
if err != nil {
return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
}
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
Name: fmt.Sprintf("garage-%d", alloc.RPCPort),
Cmd: "garage",
Args: []string{"-c", childConfPath, "server"},
SigKillWait: 1 * time.Minute,
})
}
return pmuxProcConfigs, nil
}
func garageApplyLayoutDiffPmuxProcConfig(env *crypticnet.Env) pmuxlib.ProcessConfig {
return pmuxlib.ProcessConfig{
Name: "garage-apply-layout-diff",
Cmd: "bash",
Args: waitForGarageArgs(env, "bash", "garage-apply-layout-diff"),
NoRestartOn: []int{0},
}
}

View File

@ -0,0 +1,72 @@
package entrypoint
import (
crypticnet "cryptic-net"
"cryptic-net/yamlutil"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"github.com/imdario/mergo"
"gopkg.in/yaml.v3"
)
func builtinDaemonYmlPath(env *crypticnet.Env) string {
return filepath.Join(env.AppDirPath, "etc", "daemon.yml")
}
func writeBuiltinDaemonYml(env *crypticnet.Env, w io.Writer) error {
builtinDaemonYmlPath := builtinDaemonYmlPath(env)
builtinDaemonYml, err := os.ReadFile(builtinDaemonYmlPath)
if err != nil {
return fmt.Errorf("reading default daemon.yml at %q: %w", builtinDaemonYmlPath, err)
}
if _, err := w.Write(builtinDaemonYml); err != nil {
return fmt.Errorf("writing default daemon.yml: %w", err)
}
return nil
}
func writeMergedDaemonYml(env *crypticnet.Env, userDaemonYmlPath string) error {
builtinDaemonYmlPath := builtinDaemonYmlPath(env)
var fullDaemonYml map[string]interface{}
if err := yamlutil.LoadYamlFile(&fullDaemonYml, builtinDaemonYmlPath); err != nil {
return fmt.Errorf("parsing builtin daemon.yml file: %w", err)
}
if userDaemonYmlPath != "" {
var daemonYml map[string]interface{}
if err := yamlutil.LoadYamlFile(&daemonYml, userDaemonYmlPath); err != nil {
return fmt.Errorf("parsing %q: %w", userDaemonYmlPath, err)
}
err := mergo.Merge(&fullDaemonYml, daemonYml, mergo.WithOverride)
if err != nil {
return fmt.Errorf("merging contents of file %q: %w", userDaemonYmlPath, err)
}
}
fullDaemonYmlB, err := yaml.Marshal(fullDaemonYml)
if err != nil {
return fmt.Errorf("yaml marshaling daemon config: %w", err)
}
daemonYmlPath := filepath.Join(env.RuntimeDirPath, "daemon.yml")
if err := ioutil.WriteFile(daemonYmlPath, fullDaemonYmlB, 0400); err != nil {
return fmt.Errorf("writing daemon.yml file to %q: %w", daemonYmlPath, err)
}
return nil
}

View File

@ -1,129 +0,0 @@
package garage_entrypoint
import (
"fmt"
"log"
"net"
"os"
"path/filepath"
"strconv"
"time"
crypticnet "cryptic-net"
"cryptic-net/garage"
"github.com/cryptic-io/pmux/pmuxlib"
)
func writeChildConf(
env *crypticnet.Env,
alloc crypticnet.DaemonYmlStorageAllocation,
) (string, error) {
if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
}
thisHost := env.Bootstrap.ThisHost()
peer := garage.Peer{
IP: thisHost.Nebula.IP,
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
}
pubKey, privKey := peer.RPCPeerKey()
nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
}
garageTomlPath := filepath.Join(
env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
)
err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
MetaPath: alloc.MetaPath,
DataPath: alloc.DataPath,
RPCSecret: env.Bootstrap.GarageRPCSecret,
RPCAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
APIAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
WebAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.WebPort)),
BootstrapPeers: env.Bootstrap.GarageRPCPeerAddrs(),
})
if err != nil {
return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
}
return garageTomlPath, nil
}
func waitForArgs(env *crypticnet.Env, bin string, binArgs ...string) []string {
thisHost := env.Bootstrap.ThisHost()
var args []string
for _, alloc := range env.ThisDaemon().Storage.Allocations {
args = append(
args,
"wait-for",
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
"--",
)
}
args = append(args, bin)
args = append(args, binArgs...)
return args
}
func Main() {
env, err := crypticnet.ReadEnv()
if err != nil {
log.Fatalf("reading envvars: %v", err)
}
var pmuxProcConfigs []pmuxlib.ProcessConfig
for _, alloc := range env.ThisDaemon().Storage.Allocations {
childConfPath, err := writeChildConf(env, alloc)
if err != nil {
log.Fatalf("writing child config file for alloc %+v: %v", alloc, err)
}
log.Printf("wrote config file %q", childConfPath)
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
Name: fmt.Sprintf("garage-%d", alloc.RPCPort),
Cmd: "garage",
Args: []string{"-c", childConfPath, "server"},
SigKillWait: 1 * time.Minute,
})
}
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
Name: "garage-apply-layout-diff",
Cmd: "bash",
Args: waitForArgs(env, "bash", "garage-apply-layout-diff"),
NoRestartOn: []int{0},
})
pmuxlib.Run(env.Context, pmuxlib.Config{Processes: pmuxProcConfigs})
}

View File

@ -102,11 +102,6 @@ func Main() {
Proto: "tcp",
Host: "any",
},
crypticnet.ConfigFirewallRule{
Port: strconv.Itoa(alloc.WebPort),
Proto: "tcp",
Host: "any",
},
)
}

View File

@ -31,9 +31,9 @@ type DaemonYmlStorageAllocation struct {
DataPath string `yaml:"data_path"`
MetaPath string `yaml:"meta_path"`
Capacity int `yaml:"capacity"`
S3APIPort int `yaml:"api_port"` // TODO fix field name here
S3APIPort int `yaml:"s3_api_port"`
RPCPort int `yaml:"rpc_port"`
WebPort int `yaml:"web_port"`
AdminPort int `yaml:"admin_port"`
}
// DaemonYml describes the structure of the daemon.yml file.

View File

@ -0,0 +1,84 @@
package garage
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
)
// AdminClient is a helper type for performing actions against the garage admin
// interface.
type AdminClient struct {
c *http.Client
addr string
adminToken string
}
// NewAdminClient initializes and returns an AdminClient which will use the
// given address and adminToken for all requests made.
func NewAdminClient(addr, adminToken string) *AdminClient {
return &AdminClient{
c: &http.Client{
Transport: http.DefaultTransport.(*http.Transport).Clone(),
},
addr: addr,
adminToken: adminToken,
}
}
// Do performs an HTTP request with the given method (GET, POST) and path, and
// using the json marshaling of the given body as the request body (unless body
// is nil). It will JSON unmarshal the response into rcv, unless rcv is nil.
func (c *AdminClient) Do(
ctx context.Context, rcv interface{}, method, path string, body interface{},
) error {
var bodyR io.Reader
if body != nil {
bodyBuf := new(bytes.Buffer)
bodyR = bodyBuf
if err := json.NewEncoder(bodyBuf).Encode(body); err != nil {
return fmt.Errorf("json marshaling body: %w", err)
}
}
urlStr := fmt.Sprintf("http://%s%s", c.addr, path)
req, err := http.NewRequestWithContext(ctx, method, urlStr, bodyR)
if err != nil {
return fmt.Errorf("initializing request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+c.adminToken)
res, err := c.c.Do(req)
if err != nil {
return fmt.Errorf("performing http request: %w", err)
}
defer res.Body.Close()
if res.StatusCode != 200 {
return fmt.Errorf("unexpected %s response returned", res.Status)
}
if rcv == nil {
if _, err := io.Copy(io.Discard, res.Body); err != nil {
return fmt.Errorf("discarding response body: %w", err)
}
return nil
}
if err := json.NewDecoder(res.Body).Decode(rcv); err != nil {
return fmt.Errorf("decoding json response body: %w", err)
}
return nil
}

View File

@ -14,10 +14,11 @@ type GarageTomlData struct {
DataPath string
RPCSecret string
AdminToken string
RPCAddr string
APIAddr string
WebAddr string
AdminAddr string
BootstrapPeers []string
}
@ -41,9 +42,9 @@ bootstrap_peers = [{{- range .BootstrapPeers }}
api_bind_addr = "{{ .APIAddr }}"
s3_region = "garage"
[s3_web]
bind_addr = "{{ .WebAddr }}"
root_domain = ".example.com"
[admin]
api_bind_addr = "{{ .AdminAddr }}"
admin_token = "{{ .AdminToken }}"
`))

View File

@ -14,15 +14,6 @@ import (
"golang.org/x/crypto/curve25519"
)
// TODO this should one day not be hardcoded
var ipCIDRMask = func() net.IPMask {
_, ipNet, err := net.ParseCIDR("10.10.0.0/16")
if err != nil {
panic(err)
}
return ipNet.Mask
}()
// HostCert contains the certificate and private key files which will need to
// be present on a particular host. Each file is PEM encoded.
type HostCert struct {
@ -41,7 +32,7 @@ type CACert struct {
// NewHostCert generates a new key/cert for a nebula host using the CA key
// which will be found in the adminFS.
func NewHostCert(
caCert CACert, hostName, hostIP string,
caCert CACert, hostName string, ip net.IP,
) (
HostCert, error,
) {
@ -66,14 +57,9 @@ func NewHostCert(
expireAt := caCrt.Details.NotAfter.Add(-1 * time.Second)
ip := net.ParseIP(hostIP)
if ip == nil {
return HostCert{}, fmt.Errorf("invalid host ip %q", hostIP)
}
ipNet := &net.IPNet{
IP: ip,
Mask: ipCIDRMask,
subnet := caCrt.Details.Subnets[0]
if !subnet.Contains(ip) {
return HostCert{}, fmt.Errorf("invalid ip %q, not contained by network subnet %q", ip, subnet)
}
var hostPub, hostKey []byte
@ -89,7 +75,10 @@ func NewHostCert(
hostCrt := cert.NebulaCertificate{
Details: cert.NebulaCertificateDetails{
Name: hostName,
Ips: []*net.IPNet{ipNet},
Ips: []*net.IPNet{{
IP: ip,
Mask: subnet.Mask,
}},
NotBefore: time.Now(),
NotAfter: expireAt,
PublicKey: hostPub,
@ -122,7 +111,7 @@ func NewHostCert(
// NewCACert generates a CACert. The domain should be the network's root domain,
// and is included in the signing certificate's Name field.
func NewCACert(domain string) (CACert, error) {
func NewCACert(domain string, subnet *net.IPNet) (CACert, error) {
pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
@ -135,6 +124,7 @@ func NewCACert(domain string) (CACert, error) {
caCrt := cert.NebulaCertificate{
Details: cert.NebulaCertificateDetails{
Name: fmt.Sprintf("%s cryptic-net root cert", domain),
Subnets: []*net.IPNet{subnet},
NotBefore: now,
NotAfter: expireAt,
PublicKey: pubKey,