Implement admin create-network command
This required a lot of re-implementation of how garage gets interacted with, including updating cluster layout using the admin API and initialization of the global bucket key.
This commit is contained in:
Brian Picciano
parent
7a25e1b6e6
commit
41e0b56617
@ -27,8 +27,8 @@ const (
|
||||
// CreationParams are general parameters used when creating a new network. These
|
||||
// are available to all hosts within the network via their bootstrap files.
|
||||
type CreationParams struct {
|
||||
Domain string `yaml:"domain"`
|
||||
CIDRs []string `yaml:"cidrs"`
|
||||
ID string `yaml:"id"`
|
||||
Domain string `yaml:"domain"`
|
||||
}
|
||||
|
||||
// Admin is used for accessing all information contained within an admin.tgz.
|
||||
|
||||
@ -1,14 +1,22 @@
|
||||
package entrypoint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"cryptic-net/admin"
|
||||
"cryptic-net/bootstrap"
|
||||
"cryptic-net/garage"
|
||||
"cryptic-net/nebula"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/cryptic-io/pmux/pmuxlib"
|
||||
)
|
||||
|
||||
func randStr(l int) string {
|
||||
@ -40,6 +48,294 @@ func readAdmin(path string) (admin.Admin, error) {
|
||||
return admin.FromReader(f)
|
||||
}
|
||||
|
||||
func garageInitializeGlobalBucket(
|
||||
ctx context.Context,
|
||||
adminClient *garage.AdminClient,
|
||||
globalBucketCreds garage.S3APICredentials,
|
||||
) error {
|
||||
|
||||
// first attempt to import the key
|
||||
err := adminClient.Do(ctx, nil, "POST", "/v0/key/import", map[string]string{
|
||||
"accessKeyId": globalBucketCreds.ID,
|
||||
"secretAccessKey": globalBucketCreds.Secret,
|
||||
"name": "shared-global-bucket-key",
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("importing global bucket key into garage: %w", err)
|
||||
}
|
||||
|
||||
// create global bucket
|
||||
err = adminClient.Do(ctx, nil, "POST", "/v0/bucket", map[string]string{
|
||||
"globalAlias": garage.GlobalBucket,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating global bucket: %w", err)
|
||||
}
|
||||
|
||||
// retrieve newly created bucket's id
|
||||
var getBucketRes struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
err = adminClient.Do(
|
||||
ctx, &getBucketRes,
|
||||
"GET", "/v0/bucket?globalAlias="+garage.GlobalBucket, nil,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetching global bucket id: %w", err)
|
||||
}
|
||||
|
||||
// allow shared global bucket key to perform all operations
|
||||
err = adminClient.Do(ctx, nil, "POST", "/v0/bucket/allow", map[string]interface{}{
|
||||
"bucketId": getBucketRes.ID,
|
||||
"accessKeyId": globalBucketCreds.ID,
|
||||
"permissions": map[string]bool{
|
||||
"read": true,
|
||||
"write": true,
|
||||
},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("granting permissions to shared global bucket key: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var subCmdAdminCreateNetwork = subCmd{
|
||||
name: "create-network",
|
||||
descr: "Creates a new cryptic-net network, outputting the resulting admin.tgz to stdout",
|
||||
do: func(subCmdCtx subCmdCtx) error {
|
||||
|
||||
flags := subCmdCtx.flagSet(false)
|
||||
|
||||
daemonYmlPath := flags.StringP(
|
||||
"config-path", "c", "",
|
||||
"Optional path to a daemon.yml file to load configuration from.",
|
||||
)
|
||||
|
||||
dumpConfig := flags.Bool(
|
||||
"dump-config", false,
|
||||
"Write the default configuration file to stdout and exit.",
|
||||
)
|
||||
|
||||
domain := flags.StringP(
|
||||
"domain", "d", "",
|
||||
"Domain name that should be used as the root domain in the network.",
|
||||
)
|
||||
|
||||
subnetStr := flags.StringP(
|
||||
"subnet", "s", "",
|
||||
"CIDR which denotes the subnet that IPs hosts on the network can be assigned.",
|
||||
)
|
||||
|
||||
hostName := flags.StringP(
|
||||
"name", "n", "",
|
||||
"Name of the host which will be the first host in the network",
|
||||
)
|
||||
|
||||
if err := flags.Parse(subCmdCtx.args); err != nil {
|
||||
return fmt.Errorf("parsing flags: %w", err)
|
||||
}
|
||||
|
||||
env := subCmdCtx.env
|
||||
|
||||
if *dumpConfig {
|
||||
return writeBuiltinDaemonYml(env, os.Stdout)
|
||||
}
|
||||
|
||||
if *domain == "" || *subnetStr == "" || *hostName == "" {
|
||||
return errors.New("--domain, --subnet, and --name are required")
|
||||
}
|
||||
|
||||
*domain = strings.TrimRight(strings.TrimLeft(*domain, "."), ".")
|
||||
|
||||
ip, subnet, err := net.ParseCIDR(*subnetStr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing %q as a CIDR: %w", *subnetStr, err)
|
||||
}
|
||||
|
||||
if err := validateHostName(*hostName); err != nil {
|
||||
return fmt.Errorf("invalid hostname %q: %w", *hostName, err)
|
||||
}
|
||||
|
||||
adminCreationParams := admin.CreationParams{
|
||||
ID: randStr(32),
|
||||
Domain: *domain,
|
||||
}
|
||||
|
||||
{
|
||||
runtimeDirPath := env.RuntimeDirPath
|
||||
|
||||
fmt.Fprintf(os.Stderr, "will use runtime directory %q for temporary state\n", runtimeDirPath)
|
||||
|
||||
if err := os.MkdirAll(runtimeDirPath, 0700); err != nil {
|
||||
return fmt.Errorf("creating directory %q: %w", runtimeDirPath, err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
fmt.Fprintf(os.Stderr, "cleaning up runtime directory %q\n", runtimeDirPath)
|
||||
if err := os.RemoveAll(runtimeDirPath); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error removing temporary directory %q: %v", runtimeDirPath, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if err := writeMergedDaemonYml(env, *daemonYmlPath); err != nil {
|
||||
return fmt.Errorf("merging and writing daemon.yml file: %w", err)
|
||||
}
|
||||
|
||||
daemon := env.ThisDaemon()
|
||||
|
||||
if len(daemon.Storage.Allocations) < 3 {
|
||||
return fmt.Errorf("daemon.yml with at least 3 allocations was not provided")
|
||||
}
|
||||
|
||||
nebulaCACert, err := nebula.NewCACert(*domain, subnet)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating nebula CA cert: %w", err)
|
||||
}
|
||||
|
||||
nebulaHostCert, err := nebula.NewHostCert(nebulaCACert, *hostName, ip)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating nebula cert for host: %w", err)
|
||||
}
|
||||
|
||||
env.Bootstrap = bootstrap.Bootstrap{
|
||||
AdminCreationParams: adminCreationParams,
|
||||
Hosts: map[string]bootstrap.Host{
|
||||
*hostName: bootstrap.Host{
|
||||
Name: *hostName,
|
||||
Nebula: bootstrap.NebulaHost{
|
||||
IP: ip.String(),
|
||||
},
|
||||
},
|
||||
},
|
||||
HostName: *hostName,
|
||||
NebulaHostCert: nebulaHostCert,
|
||||
GarageRPCSecret: randStr(32),
|
||||
GarageGlobalBucketS3APICredentials: garage.NewS3APICredentials(),
|
||||
}
|
||||
|
||||
// this will also write the bootstrap file
|
||||
if err := mergeDaemonIntoBootstrap(env); err != nil {
|
||||
return fmt.Errorf("merging daemon.yml into bootstrap data: %w", err)
|
||||
}
|
||||
|
||||
// TODO this can be gotten rid of once nebula-entrypoint is rolled into
|
||||
// daemon itself
|
||||
for key, val := range env.ToMap() {
|
||||
if err := os.Setenv(key, val); err != nil {
|
||||
return fmt.Errorf("failed to set %q to %q: %w", key, val, err)
|
||||
}
|
||||
}
|
||||
|
||||
garageChildrenPmuxProcConfigs, err := garageChildrenPmuxProcConfigs(env)
|
||||
if err != nil {
|
||||
return fmt.Errorf("generating garage children configs: %w", err)
|
||||
}
|
||||
|
||||
pmuxConfig := pmuxlib.Config{
|
||||
Processes: append(
|
||||
[]pmuxlib.ProcessConfig{
|
||||
nebulaEntrypointPmuxProcConfig(),
|
||||
},
|
||||
garageChildrenPmuxProcConfigs...,
|
||||
),
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(env.Context)
|
||||
pmuxDoneCh := make(chan struct{})
|
||||
|
||||
log.Printf("starting child processes")
|
||||
go func() {
|
||||
pmuxlib.Run(ctx, pmuxConfig)
|
||||
close(pmuxDoneCh)
|
||||
}()
|
||||
|
||||
defer func() {
|
||||
cancel()
|
||||
log.Printf("waiting for child processes to exit")
|
||||
<-pmuxDoneCh
|
||||
}()
|
||||
|
||||
var garageAdminClient *garage.AdminClient
|
||||
garageAdminClients := map[string]*garage.AdminClient{}
|
||||
|
||||
for _, alloc := range daemon.Storage.Allocations {
|
||||
|
||||
garageAdminAddr := net.JoinHostPort(ip.String(), strconv.Itoa(alloc.AdminPort))
|
||||
|
||||
garageAdminClient = garage.NewAdminClient(
|
||||
garageAdminAddr,
|
||||
env.Bootstrap.GarageAdminToken,
|
||||
)
|
||||
|
||||
garageAdminClients[garageAdminAddr] = garageAdminClient
|
||||
}
|
||||
|
||||
log.Printf("waiting for garage instances to come online")
|
||||
for garageAdminAddr, garageAdminClient := range garageAdminClients {
|
||||
if err := garageAdminClient.Wait(ctx); err != nil {
|
||||
return fmt.Errorf("waiting for garage instance %q to start up: %w", garageAdminAddr, err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("applying initial garage layout")
|
||||
err = garageApplyLayout(
|
||||
ctx,
|
||||
garageAdminClient,
|
||||
*hostName, ip.String(),
|
||||
daemon.Storage.Allocations,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("applying initial garage layout: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("initializing garage shared global bucket")
|
||||
err = garageInitializeGlobalBucket(
|
||||
ctx,
|
||||
garageAdminClient,
|
||||
env.Bootstrap.GarageGlobalBucketS3APICredentials,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("initializing garage shared global bucket: %w", err)
|
||||
}
|
||||
|
||||
garageS3Client, err := env.Bootstrap.GlobalBucketS3APIClient()
|
||||
if err != nil {
|
||||
return fmt.Errorf("initializing garage shared global bucket client: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("writing data for this host into garage")
|
||||
err = bootstrap.PutGarageBoostrapHost(ctx, garageS3Client, env.Bootstrap.ThisHost())
|
||||
if err != nil {
|
||||
return fmt.Errorf("putting host data into garage: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("cluster initialized successfully, writing admin.tgz to stdout")
|
||||
|
||||
err = admin.Admin{
|
||||
CreationParams: adminCreationParams,
|
||||
NebulaCACert: nebulaCACert,
|
||||
GarageRPCSecret: env.Bootstrap.GarageRPCSecret,
|
||||
GarageGlobalBucketS3APICredentials: env.Bootstrap.GarageGlobalBucketS3APICredentials,
|
||||
GarageAdminBucketS3APICredentials: garage.NewS3APICredentials(),
|
||||
}.WriteTo(os.Stdout)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing admin.tgz to stdout")
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var subCmdAdminMakeBootstrap = subCmd{
|
||||
name: "make-bootstrap",
|
||||
descr: "Creates a new bootstrap.tgz file for a particular host and writes it to stdout",
|
||||
@ -92,7 +388,12 @@ var subCmdAdminMakeBootstrap = subCmd{
|
||||
return fmt.Errorf("couldn't find host into for %q in garage, has `cryptic-net hosts add` been run yet?", *name)
|
||||
}
|
||||
|
||||
nebulaHostCert, err := nebula.NewHostCert(adm.NebulaCACert, host.Name, host.Nebula.IP)
|
||||
ip := net.ParseIP(host.Nebula.IP)
|
||||
if ip == nil {
|
||||
return fmt.Errorf("invalid IP stored with host %q: %q", *name, host.Nebula.IP)
|
||||
}
|
||||
|
||||
nebulaHostCert, err := nebula.NewHostCert(adm.NebulaCACert, host.Name, ip)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating new nebula host key/cert: %w", err)
|
||||
}
|
||||
|
||||
@ -4,14 +4,10 @@ import (
|
||||
"bytes"
|
||||
crypticnet "cryptic-net"
|
||||
"cryptic-net/bootstrap"
|
||||
"cryptic-net/garage"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/cryptic-io/pmux/pmuxlib"
|
||||
)
|
||||
@ -83,29 +79,6 @@ func waitForNebulaArgs(env *crypticnet.Env, args ...string) []string {
|
||||
return append([]string{"wait-for-ip", thisHost.Nebula.IP}, args...)
|
||||
}
|
||||
|
||||
func waitForGarageArgs(env *crypticnet.Env, args ...string) []string {
|
||||
|
||||
thisHost := env.Bootstrap.ThisHost()
|
||||
allocs := env.ThisDaemon().Storage.Allocations
|
||||
|
||||
if len(allocs) == 0 {
|
||||
return waitForNebulaArgs(env, args...)
|
||||
}
|
||||
|
||||
var preArgs []string
|
||||
|
||||
for _, alloc := range allocs {
|
||||
preArgs = append(
|
||||
preArgs,
|
||||
"wait-for",
|
||||
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
|
||||
"--",
|
||||
)
|
||||
}
|
||||
|
||||
return append(preArgs, args...)
|
||||
}
|
||||
|
||||
func nebulaEntrypointPmuxProcConfig() pmuxlib.ProcessConfig {
|
||||
return pmuxlib.ProcessConfig{
|
||||
Name: "nebula",
|
||||
@ -115,91 +88,3 @@ func nebulaEntrypointPmuxProcConfig() pmuxlib.ProcessConfig {
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func garageWriteChildConf(
|
||||
env *crypticnet.Env,
|
||||
alloc crypticnet.DaemonYmlStorageAllocation,
|
||||
) (
|
||||
string, error,
|
||||
) {
|
||||
|
||||
if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
|
||||
return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
|
||||
}
|
||||
|
||||
thisHost := env.Bootstrap.ThisHost()
|
||||
|
||||
peer := garage.Peer{
|
||||
IP: thisHost.Nebula.IP,
|
||||
RPCPort: alloc.RPCPort,
|
||||
S3APIPort: alloc.S3APIPort,
|
||||
}
|
||||
|
||||
pubKey, privKey := peer.RPCPeerKey()
|
||||
|
||||
nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
|
||||
nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
|
||||
|
||||
if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
|
||||
return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
|
||||
|
||||
} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
|
||||
return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
|
||||
}
|
||||
|
||||
garageTomlPath := filepath.Join(
|
||||
env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
|
||||
)
|
||||
|
||||
err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
|
||||
MetaPath: alloc.MetaPath,
|
||||
DataPath: alloc.DataPath,
|
||||
|
||||
RPCSecret: env.Bootstrap.GarageRPCSecret,
|
||||
AdminToken: env.Bootstrap.GarageAdminToken,
|
||||
|
||||
RPCAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
|
||||
APIAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
|
||||
AdminAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.AdminPort)),
|
||||
|
||||
BootstrapPeers: env.Bootstrap.GarageRPCPeerAddrs(),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
|
||||
}
|
||||
|
||||
return garageTomlPath, nil
|
||||
}
|
||||
|
||||
func garageChildrenPmuxProcConfigs(env *crypticnet.Env) ([]pmuxlib.ProcessConfig, error) {
|
||||
|
||||
var pmuxProcConfigs []pmuxlib.ProcessConfig
|
||||
|
||||
for _, alloc := range env.ThisDaemon().Storage.Allocations {
|
||||
|
||||
childConfPath, err := garageWriteChildConf(env, alloc)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
|
||||
}
|
||||
|
||||
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
|
||||
Name: fmt.Sprintf("garage-%d", alloc.RPCPort),
|
||||
Cmd: "garage",
|
||||
Args: []string{"-c", childConfPath, "server"},
|
||||
SigKillWait: 1 * time.Minute,
|
||||
})
|
||||
}
|
||||
|
||||
return pmuxProcConfigs, nil
|
||||
}
|
||||
|
||||
func garageApplyLayoutDiffPmuxProcConfig(env *crypticnet.Env) pmuxlib.ProcessConfig {
|
||||
return pmuxlib.ProcessConfig{
|
||||
Name: "garage-apply-layout-diff",
|
||||
Cmd: "bash",
|
||||
Args: waitForGarageArgs(env, "bash", "garage-apply-layout-diff"),
|
||||
NoRestartOn: []int{0},
|
||||
}
|
||||
}
|
||||
|
||||
189
go-workspace/src/cmd/entrypoint/garage_util.go
Normal file
189
go-workspace/src/cmd/entrypoint/garage_util.go
Normal file
@ -0,0 +1,189 @@
|
||||
package entrypoint
|
||||
|
||||
import (
|
||||
"context"
|
||||
crypticnet "cryptic-net"
|
||||
"cryptic-net/garage"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/cryptic-io/pmux/pmuxlib"
|
||||
)
|
||||
|
||||
func waitForGarageArgs(env *crypticnet.Env, args ...string) []string {
|
||||
|
||||
thisHost := env.Bootstrap.ThisHost()
|
||||
allocs := env.ThisDaemon().Storage.Allocations
|
||||
|
||||
if len(allocs) == 0 {
|
||||
return waitForNebulaArgs(env, args...)
|
||||
}
|
||||
|
||||
var preArgs []string
|
||||
|
||||
for _, alloc := range allocs {
|
||||
preArgs = append(
|
||||
preArgs,
|
||||
"wait-for",
|
||||
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
|
||||
"--",
|
||||
)
|
||||
}
|
||||
|
||||
return append(preArgs, args...)
|
||||
}
|
||||
|
||||
func garageWriteChildConf(
|
||||
env *crypticnet.Env,
|
||||
alloc crypticnet.DaemonYmlStorageAllocation,
|
||||
) (
|
||||
string, error,
|
||||
) {
|
||||
|
||||
if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
|
||||
return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
|
||||
}
|
||||
|
||||
thisHost := env.Bootstrap.ThisHost()
|
||||
|
||||
peer := garage.Peer{
|
||||
IP: thisHost.Nebula.IP,
|
||||
RPCPort: alloc.RPCPort,
|
||||
S3APIPort: alloc.S3APIPort,
|
||||
}
|
||||
|
||||
pubKey, privKey := peer.RPCPeerKey()
|
||||
|
||||
nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
|
||||
nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
|
||||
|
||||
if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
|
||||
return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
|
||||
|
||||
} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
|
||||
return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
|
||||
}
|
||||
|
||||
garageTomlPath := filepath.Join(
|
||||
env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
|
||||
)
|
||||
|
||||
err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
|
||||
MetaPath: alloc.MetaPath,
|
||||
DataPath: alloc.DataPath,
|
||||
|
||||
RPCSecret: env.Bootstrap.GarageRPCSecret,
|
||||
AdminToken: env.Bootstrap.GarageAdminToken,
|
||||
|
||||
RPCAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
|
||||
APIAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
|
||||
AdminAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.AdminPort)),
|
||||
|
||||
BootstrapPeers: env.Bootstrap.GarageRPCPeerAddrs(),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
|
||||
}
|
||||
|
||||
return garageTomlPath, nil
|
||||
}
|
||||
|
||||
func garageChildrenPmuxProcConfigs(env *crypticnet.Env) ([]pmuxlib.ProcessConfig, error) {
|
||||
|
||||
var pmuxProcConfigs []pmuxlib.ProcessConfig
|
||||
|
||||
for _, alloc := range env.ThisDaemon().Storage.Allocations {
|
||||
|
||||
childConfPath, err := garageWriteChildConf(env, alloc)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
|
||||
}
|
||||
|
||||
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
|
||||
Name: fmt.Sprintf("garage-%d", alloc.RPCPort),
|
||||
Cmd: "garage",
|
||||
Args: []string{"-c", childConfPath, "server"},
|
||||
SigKillWait: 1 * time.Minute,
|
||||
})
|
||||
}
|
||||
|
||||
return pmuxProcConfigs, nil
|
||||
}
|
||||
|
||||
func garageApplyLayoutDiffPmuxProcConfig(env *crypticnet.Env) pmuxlib.ProcessConfig {
|
||||
return pmuxlib.ProcessConfig{
|
||||
Name: "garage-apply-layout-diff",
|
||||
Cmd: "bash",
|
||||
Args: waitForGarageArgs(env, "bash", "garage-apply-layout-diff"),
|
||||
NoRestartOn: []int{0},
|
||||
}
|
||||
}
|
||||
|
||||
func garageApplyLayout(
|
||||
ctx context.Context,
|
||||
adminClient *garage.AdminClient,
|
||||
hostName, ipStr string,
|
||||
allocs []crypticnet.DaemonYmlStorageAllocation,
|
||||
) error {
|
||||
|
||||
type peerLayout struct {
|
||||
Capacity int `json:"capacity"`
|
||||
Zone string `json:"zone"`
|
||||
Tags []string `json:"tags"`
|
||||
}
|
||||
|
||||
{
|
||||
clusterLayout := map[string]peerLayout{}
|
||||
|
||||
for _, alloc := range allocs {
|
||||
|
||||
peer := garage.Peer{
|
||||
IP: ipStr,
|
||||
RPCPort: alloc.RPCPort,
|
||||
S3APIPort: alloc.S3APIPort,
|
||||
}
|
||||
|
||||
clusterLayout[peer.RPCPeerID()] = peerLayout{
|
||||
Capacity: alloc.Capacity / 100,
|
||||
Zone: hostName,
|
||||
}
|
||||
}
|
||||
|
||||
err := adminClient.Do(ctx, nil, "POST", "/v0/layout", clusterLayout)
|
||||
if err != nil {
|
||||
return fmt.Errorf("staging layout changes: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var clusterLayout struct {
|
||||
Version int `json:"version"`
|
||||
StagedRoleChanges map[string]peerLayout `json:"stagedRoleChanges"`
|
||||
}
|
||||
|
||||
if err := adminClient.Do(ctx, &clusterLayout, "GET", "/v0/layout", nil); err != nil {
|
||||
return fmt.Errorf("retrieving staged layout change: %w", err)
|
||||
}
|
||||
|
||||
if len(clusterLayout.StagedRoleChanges) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
applyClusterLayout := struct {
|
||||
Version int `json:"version"`
|
||||
}{
|
||||
Version: clusterLayout.Version + 1,
|
||||
}
|
||||
|
||||
err := adminClient.Do(ctx, nil, "POST", "/v0/layout/apply", applyClusterLayout)
|
||||
if err != nil {
|
||||
return fmt.Errorf("applying new layout (new version:%d): %w", applyClusterLayout.Version, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -82,3 +82,38 @@ func (c *AdminClient) Do(
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Wait will block until the instance connected to can see at least
|
||||
// ReplicationFactor-1 other garage instances. If the context is canceled it
|
||||
// will return the context error.
|
||||
func (c *AdminClient) Wait(ctx context.Context) error {
|
||||
for {
|
||||
|
||||
var clusterStatus struct {
|
||||
KnownNodes map[string]struct {
|
||||
IsUp bool `json:"is_up"`
|
||||
} `json:"knownNodes"`
|
||||
}
|
||||
|
||||
err := c.Do(ctx, &clusterStatus, "GET", "/v0/status", nil)
|
||||
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return ctxErr
|
||||
|
||||
} else if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var numUp int
|
||||
|
||||
for _, knownNode := range clusterStatus.KnownNodes {
|
||||
if knownNode.IsUp {
|
||||
numUp++
|
||||
}
|
||||
}
|
||||
|
||||
if numUp >= ReplicationFactor-1 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,12 +1,22 @@
|
||||
package garage
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
func randStr(l int) string {
|
||||
b := make([]byte, l)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
// IsKeyNotFound returns true if the given error is the result of a key not
|
||||
// being found in a bucket.
|
||||
func IsKeyNotFound(err error) bool {
|
||||
@ -24,6 +34,14 @@ type S3APICredentials struct {
|
||||
Secret string `yaml:"secret"`
|
||||
}
|
||||
|
||||
// NewS3APICredentials returns a new usable instance of S3APICredentials.
|
||||
func NewS3APICredentials() S3APICredentials {
|
||||
return S3APICredentials{
|
||||
ID: randStr(8),
|
||||
Secret: randStr(32),
|
||||
}
|
||||
}
|
||||
|
||||
// NewS3APIClient returns a minio client configured to use the given garage S3 API
|
||||
// endpoint.
|
||||
func NewS3APIClient(addr string, creds S3APICredentials) (S3APIClient, error) {
|
||||
|
||||
@ -10,4 +10,8 @@ const (
|
||||
// GlobalBucket is the name of the global garage bucket which is
|
||||
// accessible to all hosts in the network.
|
||||
GlobalBucket = "cryptic-net-global"
|
||||
|
||||
// ReplicationFactor indicates the replication factor set on the garage
|
||||
// cluster. We currently only support a factor of 3.
|
||||
ReplicationFactor = 3
|
||||
)
|
||||
|
||||
@ -14,15 +14,6 @@ import (
|
||||
"golang.org/x/crypto/curve25519"
|
||||
)
|
||||
|
||||
// TODO this should one day not be hardcoded
|
||||
var ipCIDRMask = func() net.IPMask {
|
||||
_, ipNet, err := net.ParseCIDR("10.10.0.0/16")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return ipNet.Mask
|
||||
}()
|
||||
|
||||
// HostCert contains the certificate and private key files which will need to
|
||||
// be present on a particular host. Each file is PEM encoded.
|
||||
type HostCert struct {
|
||||
@ -41,7 +32,7 @@ type CACert struct {
|
||||
// NewHostCert generates a new key/cert for a nebula host using the CA key
|
||||
// which will be found in the adminFS.
|
||||
func NewHostCert(
|
||||
caCert CACert, hostName, hostIP string,
|
||||
caCert CACert, hostName string, ip net.IP,
|
||||
) (
|
||||
HostCert, error,
|
||||
) {
|
||||
@ -66,14 +57,9 @@ func NewHostCert(
|
||||
|
||||
expireAt := caCrt.Details.NotAfter.Add(-1 * time.Second)
|
||||
|
||||
ip := net.ParseIP(hostIP)
|
||||
if ip == nil {
|
||||
return HostCert{}, fmt.Errorf("invalid host ip %q", hostIP)
|
||||
}
|
||||
|
||||
ipNet := &net.IPNet{
|
||||
IP: ip,
|
||||
Mask: ipCIDRMask,
|
||||
subnet := caCrt.Details.Subnets[0]
|
||||
if !subnet.Contains(ip) {
|
||||
return HostCert{}, fmt.Errorf("invalid ip %q, not contained by network subnet %q", ip, subnet)
|
||||
}
|
||||
|
||||
var hostPub, hostKey []byte
|
||||
@ -88,8 +74,11 @@ func NewHostCert(
|
||||
|
||||
hostCrt := cert.NebulaCertificate{
|
||||
Details: cert.NebulaCertificateDetails{
|
||||
Name: hostName,
|
||||
Ips: []*net.IPNet{ipNet},
|
||||
Name: hostName,
|
||||
Ips: []*net.IPNet{{
|
||||
IP: ip,
|
||||
Mask: subnet.Mask,
|
||||
}},
|
||||
NotBefore: time.Now(),
|
||||
NotAfter: expireAt,
|
||||
PublicKey: hostPub,
|
||||
@ -122,7 +111,7 @@ func NewHostCert(
|
||||
|
||||
// NewCACert generates a CACert. The domain should be the network's root domain,
|
||||
// and is included in the signing certificate's Name field.
|
||||
func NewCACert(domain string) (CACert, error) {
|
||||
func NewCACert(domain string, subnet *net.IPNet) (CACert, error) {
|
||||
|
||||
pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
@ -135,6 +124,7 @@ func NewCACert(domain string) (CACert, error) {
|
||||
caCrt := cert.NebulaCertificate{
|
||||
Details: cert.NebulaCertificateDetails{
|
||||
Name: fmt.Sprintf("%s cryptic-net root cert", domain),
|
||||
Subnets: []*net.IPNet{subnet},
|
||||
NotBefore: now,
|
||||
NotAfter: expireAt,
|
||||
PublicKey: pubKey,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user