Brian Picciano
68f417b5ba
This required switching all garage admin API calls to the new v1 versions, and redoing how the global bucket key is created so it is created via the "create key" API call.
410 lines
11 KiB
Go
410 lines
11 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"isle/admin"
|
|
"isle/bootstrap"
|
|
"isle/daemon"
|
|
"isle/garage"
|
|
"isle/nebula"
|
|
"net"
|
|
"os"
|
|
"strings"
|
|
|
|
"code.betamike.com/micropelago/pmux/pmuxlib"
|
|
"github.com/mediocregopher/mediocre-go-lib/v2/mlog"
|
|
)
|
|
|
|
func randStr(l int) string {
|
|
b := make([]byte, l)
|
|
if _, err := rand.Read(b); err != nil {
|
|
panic(err)
|
|
}
|
|
return hex.EncodeToString(b)
|
|
}
|
|
|
|
func readAdmin(path string) (admin.Admin, error) {
|
|
|
|
if path == "-" {
|
|
|
|
adm, err := admin.FromReader(os.Stdin)
|
|
if err != nil {
|
|
return admin.Admin{}, fmt.Errorf("parsing admin.json from stdin: %w", err)
|
|
}
|
|
|
|
return adm, nil
|
|
}
|
|
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return admin.Admin{}, fmt.Errorf("opening file: %w", err)
|
|
}
|
|
defer f.Close()
|
|
|
|
return admin.FromReader(f)
|
|
}
|
|
|
|
var subCmdAdminCreateNetwork = subCmd{
|
|
name: "create-network",
|
|
descr: "Creates a new isle network, outputting the resulting admin.json to stdout",
|
|
do: func(subCmdCtx subCmdCtx) error {
|
|
|
|
flags := subCmdCtx.flagSet(false)
|
|
|
|
daemonConfigPath := flags.StringP(
|
|
"config-path", "c", "",
|
|
"Optional path to a daemon.yml file to load configuration from.",
|
|
)
|
|
|
|
dumpConfig := flags.Bool(
|
|
"dump-config", false,
|
|
"Write the default configuration file to stdout and exit.",
|
|
)
|
|
|
|
name := flags.StringP(
|
|
"name", "n", "",
|
|
"Human-readable name to identify the network as.",
|
|
)
|
|
|
|
domain := flags.StringP(
|
|
"domain", "d", "",
|
|
"Domain name that should be used as the root domain in the network.",
|
|
)
|
|
|
|
ipNetStr := flags.StringP(
|
|
"ip-net", "i", "",
|
|
`IP+prefix (e.g. "10.10.0.1/16") which denotes the IP of this host, which will be the first host in the network, and the range of IPs which other hosts in the network can be assigned`,
|
|
)
|
|
|
|
hostName := flags.StringP(
|
|
"hostname", "h", "",
|
|
"Name of this host, which will be the first host in the network",
|
|
)
|
|
|
|
logLevelStr := flags.StringP(
|
|
"log-level", "l", "info",
|
|
`Maximum log level which should be output. Values can be "debug", "info", "warn", "error", "fatal". Does not apply to sub-processes`,
|
|
)
|
|
|
|
if err := flags.Parse(subCmdCtx.args); err != nil {
|
|
return fmt.Errorf("parsing flags: %w", err)
|
|
}
|
|
|
|
ctx := subCmdCtx.ctx
|
|
|
|
if *dumpConfig {
|
|
return daemon.CopyDefaultConfig(os.Stdout, envAppDirPath)
|
|
}
|
|
|
|
if *name == "" || *domain == "" || *ipNetStr == "" || *hostName == "" {
|
|
return errors.New("--name, --domain, --ip-net, and --hostname are required")
|
|
}
|
|
|
|
logLevel := mlog.LevelFromString(*logLevelStr)
|
|
if logLevel == nil {
|
|
return fmt.Errorf("couldn't parse log level %q", *logLevelStr)
|
|
}
|
|
|
|
logger := subCmdCtx.logger.WithMaxLevel(logLevel.Int())
|
|
|
|
*domain = strings.TrimRight(strings.TrimLeft(*domain, "."), ".")
|
|
|
|
ip, subnet, err := net.ParseCIDR(*ipNetStr)
|
|
if err != nil {
|
|
return fmt.Errorf("parsing %q as a CIDR: %w", *ipNetStr, err)
|
|
}
|
|
|
|
if err := validateHostName(*hostName); err != nil {
|
|
return fmt.Errorf("invalid hostname %q: %w", *hostName, err)
|
|
}
|
|
|
|
runtimeDirCleanup, err := setupAndLockRuntimeDir(ctx, logger)
|
|
if err != nil {
|
|
return fmt.Errorf("setting up runtime directory: %w", err)
|
|
}
|
|
defer runtimeDirCleanup()
|
|
|
|
daemonConfig, err := daemon.LoadConfig(envAppDirPath, *daemonConfigPath)
|
|
if err != nil {
|
|
return fmt.Errorf("loading daemon config: %w", err)
|
|
}
|
|
|
|
if len(daemonConfig.Storage.Allocations) < 3 {
|
|
return fmt.Errorf("daemon config with at least 3 allocations was not provided")
|
|
}
|
|
|
|
nebulaCACreds, err := nebula.NewCACredentials(*domain, subnet)
|
|
if err != nil {
|
|
return fmt.Errorf("creating nebula CA cert: %w", err)
|
|
}
|
|
|
|
adminCreationParams := admin.CreationParams{
|
|
ID: randStr(32),
|
|
Name: *name,
|
|
Domain: *domain,
|
|
}
|
|
|
|
garageBootstrap := bootstrap.Garage{
|
|
RPCSecret: randStr(32),
|
|
AdminToken: randStr(32),
|
|
}
|
|
|
|
hostBootstrap, err := bootstrap.New(
|
|
nebulaCACreds,
|
|
adminCreationParams,
|
|
garageBootstrap,
|
|
*hostName,
|
|
ip,
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("initializing bootstrap data: %w", err)
|
|
}
|
|
|
|
if hostBootstrap, daemonConfig, err = coalesceDaemonConfigAndBootstrap(hostBootstrap, daemonConfig); err != nil {
|
|
return fmt.Errorf("merging daemon config into bootstrap data: %w", err)
|
|
}
|
|
|
|
nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(hostBootstrap, daemonConfig)
|
|
if err != nil {
|
|
return fmt.Errorf("generating nebula config: %w", err)
|
|
}
|
|
|
|
garagePmuxProcConfigs, err := garagePmuxProcConfigs(hostBootstrap, daemonConfig)
|
|
if err != nil {
|
|
return fmt.Errorf("generating garage configs: %w", err)
|
|
}
|
|
|
|
pmuxConfig := pmuxlib.Config{
|
|
Processes: append(
|
|
[]pmuxlib.ProcessConfig{
|
|
nebulaPmuxProcConfig,
|
|
},
|
|
garagePmuxProcConfigs...,
|
|
),
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
pmuxDoneCh := make(chan struct{})
|
|
|
|
logger.Info(ctx, "starting child processes")
|
|
go func() {
|
|
// NOTE both stdout and stderr are sent to stderr, so that the user
|
|
// can pipe the resulting admin.json to stdout.
|
|
pmuxlib.Run(ctx, os.Stderr, os.Stderr, pmuxConfig)
|
|
close(pmuxDoneCh)
|
|
}()
|
|
|
|
defer func() {
|
|
cancel()
|
|
logger.Info(ctx, "waiting for child processes to exit")
|
|
<-pmuxDoneCh
|
|
}()
|
|
|
|
logger.Info(ctx, "waiting for garage instances to come online")
|
|
if err := waitForGarageAndNebula(ctx, logger, hostBootstrap, daemonConfig); err != nil {
|
|
return fmt.Errorf("waiting for garage to start up: %w", err)
|
|
}
|
|
|
|
logger.Info(ctx, "applying initial garage layout")
|
|
if err := garageApplyLayout(ctx, logger, hostBootstrap, daemonConfig); err != nil {
|
|
return fmt.Errorf("applying initial garage layout: %w", err)
|
|
}
|
|
|
|
logger.Info(ctx, "initializing garage shared global bucket")
|
|
garageGlobalBucketCreds, err := garageInitializeGlobalBucket(
|
|
ctx, logger, hostBootstrap, daemonConfig,
|
|
)
|
|
|
|
hostBootstrap.Garage.GlobalBucketS3APICredentials = garageGlobalBucketCreds
|
|
|
|
// rewrite the bootstrap now that the global bucket creds have been
|
|
// added to it.
|
|
if err := writeBootstrapToDataDir(hostBootstrap); err != nil {
|
|
return fmt.Errorf("writing bootstrap file: %w", err)
|
|
}
|
|
|
|
if cErr := (garage.AdminClientError{}); errors.As(err, &cErr) && cErr.StatusCode == 409 {
|
|
return fmt.Errorf("shared global bucket has already been created, are the storage allocations from a previously initialized isle being used?")
|
|
|
|
} else if err != nil {
|
|
return fmt.Errorf("initializing garage shared global bucket: %w", err)
|
|
}
|
|
|
|
logger.Info(ctx, "cluster initialized successfully, writing admin.json to stdout")
|
|
|
|
adm := admin.Admin{
|
|
CreationParams: adminCreationParams,
|
|
}
|
|
adm.Nebula.CACredentials = nebulaCACreds
|
|
adm.Garage.RPCSecret = hostBootstrap.Garage.RPCSecret
|
|
adm.Garage.GlobalBucketS3APICredentials = hostBootstrap.Garage.GlobalBucketS3APICredentials
|
|
|
|
if err := adm.WriteTo(os.Stdout); err != nil {
|
|
return fmt.Errorf("writing admin.json to stdout")
|
|
}
|
|
|
|
return nil
|
|
},
|
|
}
|
|
|
|
var subCmdAdminCreateBootstrap = subCmd{
|
|
name: "create-bootstrap",
|
|
descr: "Creates a new bootstrap.json file for a particular host and writes it to stdout",
|
|
checkLock: false,
|
|
do: func(subCmdCtx subCmdCtx) error {
|
|
|
|
flags := subCmdCtx.flagSet(false)
|
|
|
|
hostName := flags.StringP(
|
|
"hostname", "h", "",
|
|
"Name of the host to generate bootstrap.json for",
|
|
)
|
|
|
|
ipStr := flags.StringP(
|
|
"ip", "i", "",
|
|
"IP of the new host",
|
|
)
|
|
|
|
adminPath := flags.StringP(
|
|
"admin-path", "a", "",
|
|
`Path to admin.json file. If the given path is "-" then stdin is used.`,
|
|
)
|
|
|
|
if err := flags.Parse(subCmdCtx.args); err != nil {
|
|
return fmt.Errorf("parsing flags: %w", err)
|
|
}
|
|
|
|
if *hostName == "" || *ipStr == "" || *adminPath == "" {
|
|
return errors.New("--hostname, --ip, and --admin-path are required")
|
|
}
|
|
|
|
if err := validateHostName(*hostName); err != nil {
|
|
return fmt.Errorf("invalid hostname %q: %w", *hostName, err)
|
|
}
|
|
|
|
ip := net.ParseIP(*ipStr)
|
|
|
|
if ip == nil {
|
|
return fmt.Errorf("invalid ip %q", *ipStr)
|
|
}
|
|
|
|
adm, err := readAdmin(*adminPath)
|
|
if err != nil {
|
|
return fmt.Errorf("reading admin.json with --admin-path of %q: %w", *adminPath, err)
|
|
}
|
|
|
|
garageBootstrap := bootstrap.Garage{
|
|
RPCSecret: adm.Garage.RPCSecret,
|
|
AdminToken: randStr(32),
|
|
GlobalBucketS3APICredentials: adm.Garage.GlobalBucketS3APICredentials,
|
|
}
|
|
|
|
newHostBootstrap, err := bootstrap.New(
|
|
adm.Nebula.CACredentials,
|
|
adm.CreationParams,
|
|
garageBootstrap,
|
|
*hostName,
|
|
ip,
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("initializing bootstrap data: %w", err)
|
|
}
|
|
|
|
hostBootstrap, err := loadHostBootstrap()
|
|
if err != nil {
|
|
return fmt.Errorf("loading host bootstrap: %w", err)
|
|
}
|
|
|
|
newHostBootstrap.Hosts = hostBootstrap.Hosts
|
|
|
|
return newHostBootstrap.WriteTo(os.Stdout)
|
|
},
|
|
}
|
|
|
|
var subCmdAdminCreateNebulaCert = subCmd{
|
|
name: "create-nebula-cert",
|
|
descr: "Creates a signed nebula certificate file and writes it to stdout",
|
|
checkLock: false,
|
|
do: func(subCmdCtx subCmdCtx) error {
|
|
|
|
flags := subCmdCtx.flagSet(false)
|
|
|
|
hostName := flags.StringP(
|
|
"hostname", "h", "",
|
|
"Name of the host to generate bootstrap.json for",
|
|
)
|
|
|
|
ipStr := flags.StringP(
|
|
"ip", "i", "",
|
|
"IP of the new host",
|
|
)
|
|
|
|
adminPath := flags.StringP(
|
|
"admin-path", "a", "",
|
|
`Path to admin.json file. If the given path is "-" then stdin is used.`,
|
|
)
|
|
|
|
pubKeyPath := flags.StringP(
|
|
"public-key-path", "p", "",
|
|
`Path to PEM file containing public key which will be embedded in the cert.`,
|
|
)
|
|
|
|
if err := flags.Parse(subCmdCtx.args); err != nil {
|
|
return fmt.Errorf("parsing flags: %w", err)
|
|
}
|
|
|
|
if *hostName == "" || *ipStr == "" || *adminPath == "" || *pubKeyPath == "" {
|
|
return errors.New("--hostname, --ip, --admin-path, and --pub-key-path are required")
|
|
}
|
|
|
|
if err := validateHostName(*hostName); err != nil {
|
|
return fmt.Errorf("invalid hostname %q: %w", *hostName, err)
|
|
}
|
|
|
|
ip := net.ParseIP(*ipStr)
|
|
|
|
if ip == nil {
|
|
return fmt.Errorf("invalid ip %q", *ipStr)
|
|
}
|
|
|
|
adm, err := readAdmin(*adminPath)
|
|
if err != nil {
|
|
return fmt.Errorf("reading admin.json with --admin-path of %q: %w", *adminPath, err)
|
|
}
|
|
|
|
hostPubPEM, err := os.ReadFile(*pubKeyPath)
|
|
if err != nil {
|
|
return fmt.Errorf("reading public key from %q: %w", *pubKeyPath, err)
|
|
}
|
|
|
|
nebulaHostCertPEM, err := nebula.NewHostCertPEM(
|
|
adm.Nebula.CACredentials, string(hostPubPEM), *hostName, ip,
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("creating cert: %w", err)
|
|
}
|
|
|
|
if _, err := os.Stdout.Write([]byte(nebulaHostCertPEM)); err != nil {
|
|
return fmt.Errorf("writing to stdout: %w", err)
|
|
}
|
|
|
|
return nil
|
|
},
|
|
}
|
|
|
|
var subCmdAdmin = subCmd{
|
|
name: "admin",
|
|
descr: "Sub-commands which only admins can run",
|
|
do: func(subCmdCtx subCmdCtx) error {
|
|
return subCmdCtx.doSubCmd(
|
|
subCmdAdminCreateNetwork,
|
|
subCmdAdminCreateBootstrap,
|
|
subCmdAdminCreateNebulaCert,
|
|
)
|
|
},
|
|
}
|