micropelago/isle
3
0
Fork 0
Code Issues 1 Pull Requests Projects Releases 5 Activity

Use secrets store for global garage bucket creds

Browse Source
This commit is contained in:
Brian Picciano 2024-07-14 12:19:39 +02:00
parent 86abdb6ae1
commit 9d5c8ea4db
8 changed files with 69 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
go/admin/admin.go
View File

@ -4,7 +4,6 @@ package admin
import (
"encoding/json"
"io"
"isle/garage"
"isle/nebula"
)
@ -23,10 +22,6 @@ type Admin struct {
Nebula struct {
CACredentials nebula.CACredentials
}
Garage struct {
GlobalBucketS3APICredentials garage.S3APICredentials
}
}
// FromReader reads an admin.json from the given io.Reader.

12
go/bootstrap/bootstrap.go
View File

@ -7,7 +7,6 @@ import (
"encoding/json"
"fmt"
"isle/admin"
"isle/garage"
"isle/nebula"
"net/netip"
"path/filepath"
@ -26,20 +25,11 @@ func AppDirPath(appDirPath string) string {
return filepath.Join(appDirPath, "share/bootstrap.json")
}
// Garage contains parameters needed to connect to and use the garage cluster.
type Garage struct {
AdminToken string
// TODO this should be part of admin.CreationParams
GlobalBucketS3APICredentials garage.S3APICredentials
}
// Bootstrap contains all information which is needed by a host daemon to join a
// network on boot.
type Bootstrap struct {
AdminCreationParams admin.CreationParams
CAPublicCredentials nebula.CAPublicCredentials
Garage Garage
PrivateCredentials nebula.HostPrivateCredentials
HostAssigned `json:"-"`
@ -53,7 +43,6 @@ type Bootstrap struct {
func New(
caCreds nebula.CACredentials,
adminCreationParams admin.CreationParams,
garage Garage,
name nebula.HostName,
ip netip.Addr,
) (
@ -79,7 +68,6 @@ func New(
return Bootstrap{
AdminCreationParams: adminCreationParams,
CAPublicCredentials: caCreds.Public,
Garage: garage,
PrivateCredentials: hostPrivCreds,
HostAssigned: assigned,
SignedHostAssigned: signedAssigned,

32
go/daemon/child_garage.go
View File

@ -19,11 +19,12 @@ func garageAdminClientLogger(logger *mlog.Logger) *mlog.Logger {
return logger.WithNamespace("garageAdminClient")
}
// NewGarageAdminClient will return an AdminClient for a local garage instance,
// newGarageAdminClient will return an AdminClient for a local garage instance,
// or it will _panic_ if there is no local instance configured.
func NewGarageAdminClient(
func newGarageAdminClient(
logger *mlog.Logger,
daemonConfig Config,
adminToken string,
hostBootstrap bootstrap.Bootstrap,
) *garage.AdminClient {
@ -35,7 +36,7 @@ func NewGarageAdminClient(
thisHost.IP().String(),
strconv.Itoa(daemonConfig.Storage.Allocations[0].AdminPort),
),
hostBootstrap.Garage.AdminToken,
adminToken,
)
}
@ -43,6 +44,7 @@ func waitForGarage(
ctx context.Context,
logger *mlog.Logger,
daemonConfig Config,
adminToken string,
hostBootstrap bootstrap.Bootstrap,
) error {
@ -64,9 +66,7 @@ func waitForGarage(
)
adminClient := garage.NewAdminClient(
adminClientLogger,
adminAddr,
hostBootstrap.Garage.AdminToken,
adminClientLogger, adminAddr, adminToken,
)
ctx := mctx.Annotate(ctx, "garageAdminAddr", adminAddr)
@ -101,7 +101,7 @@ func bootstrapGarageHostForAlloc(
}
func garageWriteChildConfig(
rpcSecret, runtimeDirPath string,
rpcSecret, runtimeDirPath, adminToken string,
hostBootstrap bootstrap.Bootstrap,
alloc ConfigStorageAllocation,
) (
@ -130,7 +130,7 @@ func garageWriteChildConfig(
DataPath: alloc.DataPath,
RPCSecret: rpcSecret,
AdminToken: hostBootstrap.Garage.AdminToken,
AdminToken: adminToken,
LocalPeer: peer,
BootstrapPeers: hostBootstrap.GaragePeers(),
@ -148,6 +148,7 @@ func garagePmuxProcConfigs(
logger *mlog.Logger,
rpcSecret, runtimeDirPath, binDirPath string,
daemonConfig Config,
adminToken string,
hostBootstrap bootstrap.Bootstrap,
) (
[]pmuxlib.ProcessConfig, error,
@ -165,7 +166,7 @@ func garagePmuxProcConfigs(
for _, alloc := range allocs {
childConfigPath, err := garageWriteChildConfig(
rpcSecret, runtimeDirPath, hostBootstrap, alloc,
rpcSecret, runtimeDirPath, adminToken, hostBootstrap, alloc,
)
if err != nil {
return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
@ -188,15 +189,18 @@ func garageApplyLayout(
ctx context.Context,
logger *mlog.Logger,
daemonConfig Config,
adminToken string,
hostBootstrap bootstrap.Bootstrap,
) error {
var (
adminClient = NewGarageAdminClient(logger, daemonConfig, hostBootstrap)
thisHost = hostBootstrap.ThisHost()
hostName = thisHost.Name
allocs = daemonConfig.Storage.Allocations
peers = make([]garage.PeerLayout, len(allocs))
adminClient = newGarageAdminClient(
logger, daemonConfig, adminToken, hostBootstrap,
)
thisHost = hostBootstrap.ThisHost()
hostName = thisHost.Name
allocs = daemonConfig.Storage.Allocations
peers = make([]garage.PeerLayout, len(allocs))
)
for i, alloc := range allocs {

7
go/daemon/child_pmux.go
View File

@ -12,6 +12,7 @@ func (c *Children) newPmuxConfig(
ctx context.Context,
garageRPCSecret, binDirPath string,
daemonConfig Config,
garageAdminToken string,
hostBootstrap bootstrap.Bootstrap,
) (
pmuxlib.Config, error,
@ -45,6 +46,7 @@ func (c *Children) newPmuxConfig(
c.opts.EnvVars.RuntimeDirPath,
binDirPath,
daemonConfig,
garageAdminToken,
hostBootstrap,
)
if err != nil {
@ -67,6 +69,7 @@ func (c *Children) newPmuxConfig(
func (c *Children) postPmuxInit(
ctx context.Context,
daemonConfig Config,
garageAdminToken string,
hostBootstrap bootstrap.Bootstrap,
) error {
c.logger.Info(ctx, "Waiting for nebula VPN to come online")
@ -75,7 +78,9 @@ func (c *Children) postPmuxInit(
}
c.logger.Info(ctx, "Waiting for garage instances to come online")
err := waitForGarage(ctx, c.logger, daemonConfig, hostBootstrap)
err := waitForGarage(
ctx, c.logger, daemonConfig, garageAdminToken, hostBootstrap,
)
if err != nil {
return fmt.Errorf("waiting for garage to start: %w", err)
}

12
go/daemon/children.go
View File

@ -32,6 +32,7 @@ func NewChildren(
binDirPath string,
secretsStore secrets.Store,
daemonConfig Config,
garageAdminToken string,
hostBootstrap bootstrap.Bootstrap,
opts *Opts,
) (
@ -55,7 +56,12 @@ func NewChildren(
}
pmuxConfig, err := c.newPmuxConfig(
ctx, garageRPCSecret, binDirPath, daemonConfig, hostBootstrap,
ctx,
garageRPCSecret,
binDirPath,
daemonConfig,
garageAdminToken,
hostBootstrap,
)
if err != nil {
return nil, fmt.Errorf("generating pmux config: %w", err)
@ -67,7 +73,9 @@ func NewChildren(
c.logger.Debug(pmuxCtx, "pmux stopped")
}()
initErr := c.postPmuxInit(ctx, daemonConfig, hostBootstrap)
initErr := c.postPmuxInit(
ctx, daemonConfig, garageAdminToken, hostBootstrap,
)
if initErr != nil {
logger.Warn(ctx, "failed to initialize Children, shutting down child processes", err)
if err := c.Shutdown(); err != nil {

70
go/daemon/daemon.go
View File

@ -11,7 +11,6 @@ import (
"io/fs"
"isle/admin"
"isle/bootstrap"
"isle/garage"
"isle/jsonutil"
"isle/nebula"
"isle/secrets"
@ -141,7 +140,8 @@ type daemon struct {
envBinDirPath string
opts *Opts
secretsStore secrets.Store
secretsStore secrets.Store
garageAdminToken string
l sync.RWMutex
state int
@ -177,11 +177,12 @@ func NewDaemon(
) {
var (
d = &daemon{
logger: logger,
daemonConfig: daemonConfig,
envBinDirPath: envBinDirPath,
opts: opts.withDefaults(),
shutdownCh: make(chan struct{}),
logger: logger,
daemonConfig: daemonConfig,
envBinDirPath: envBinDirPath,
opts: opts.withDefaults(),
garageAdminToken: randStr(32),
shutdownCh: make(chan struct{}),
}
bootstrapFilePath = bootstrap.StateDirPath(d.opts.EnvVars.StateDirPath)
)
@ -370,7 +371,11 @@ func (d *daemon) postInit(ctx context.Context) bool {
"Applying garage layout",
func(ctx context.Context) error {
return garageApplyLayout(
ctx, d.logger, d.daemonConfig, d.currBootstrap,
ctx,
d.logger,
d.daemonConfig,
d.garageAdminToken,
d.currBootstrap,
)
},
) {
@ -383,28 +388,29 @@ func (d *daemon) postInit(ctx context.Context) bool {
//
// TODO this is pretty hacky, but there doesn't seem to be a better way to
// manage it at the moment.
if d.currBootstrap.Garage.GlobalBucketS3APICredentials == (garage.S3APICredentials{}) {
currBootstrap := d.currBootstrap
_, err := getGarageS3APIGlobalBucketCredentials(ctx, d.secretsStore)
if errors.Is(err, secrets.ErrNotFound) {
if !until(
ctx,
d.logger,
"Initializing garage shared global bucket",
func(ctx context.Context) error {
garageGlobalBucketCreds, err := garageInitializeGlobalBucket(
ctx, d.logger, d.daemonConfig, d.currBootstrap,
ctx,
d.logger,
d.daemonConfig,
d.garageAdminToken,
d.currBootstrap,
)
if err != nil {
return fmt.Errorf("initializing global bucket: %w", err)
}
currBootstrap.Garage.GlobalBucketS3APICredentials = garageGlobalBucketCreds
d.logger.Info(ctx, "Writing bootstrap to state directory")
err = writeBootstrapToStateDir(
d.opts.EnvVars.StateDirPath, currBootstrap,
err = setGarageS3APIGlobalBucketCredentials(
ctx, d.secretsStore, garageGlobalBucketCreds,
)
if err != nil {
return fmt.Errorf("writing bootstrap to state dir: %w", err)
return fmt.Errorf("storing global bucket creds: %w", err)
}
return nil
@ -412,10 +418,6 @@ func (d *daemon) postInit(ctx context.Context) bool {
) {
return false
}
d.l.Lock()
d.currBootstrap = currBootstrap
d.l.Unlock()
}
if !until(
@ -461,6 +463,7 @@ func (d *daemon) restartLoop(ctx context.Context, readyCh chan<- struct{}) {
d.envBinDirPath,
d.secretsStore,
d.daemonConfig,
d.garageAdminToken,
d.currBootstrap,
d.opts,
)
@ -533,13 +536,9 @@ func (d *daemon) CreateNetwork(
},
}
garageBootstrap = bootstrap.Garage{
AdminToken: randStr(32),
}
garageRPCSecret = randStr(32)
)
garageRPCSecret := randStr(32)
err = setGarageRPCSecret(ctx, d.secretsStore, garageRPCSecret)
if err != nil {
return admin.Admin{}, fmt.Errorf("storing garage RPC secret: %w", err)
@ -548,7 +547,6 @@ func (d *daemon) CreateNetwork(
hostBootstrap, err := bootstrap.New(
nebulaCACreds,
adm.CreationParams,
garageBootstrap,
hostName,
ipNet.FirstAddr(),
)
@ -584,16 +582,7 @@ func (d *daemon) CreateNetwork(
return adm, ctx.Err()
}
// As part of postInit, which is called prior to ready(), the restartLoop
// will check if the global bucket creds have been created yet or not, and
// create them if so. So once ready() is called we can get the created creds
// from the currBootstrap
d.l.RLock()
garageGlobalBucketCreds := d.currBootstrap.Garage.GlobalBucketS3APICredentials
d.l.RUnlock()
adm.Nebula.CACredentials = nebulaCACreds
adm.Garage.GlobalBucketS3APICredentials = garageGlobalBucketCreds
return adm, nil
}
@ -697,13 +686,6 @@ func (d *daemon) CreateHost(
JoiningBootstrap, error,
) {
var (
garageGlobalBucketS3APICreds = currBootstrap.Garage.GlobalBucketS3APICredentials
garageBootstrap = bootstrap.Garage{
AdminToken: randStr(32),
GlobalBucketS3APICredentials: garageGlobalBucketS3APICreds,
}
joiningBootstrap JoiningBootstrap
err error
)
@ -711,7 +693,6 @@ func (d *daemon) CreateHost(
joiningBootstrap.Bootstrap, err = bootstrap.New(
makeCACreds(currBootstrap, caSigningPrivateKey),
currBootstrap.AdminCreationParams,
garageBootstrap,
hostName,
ip,
)
@ -726,6 +707,7 @@ func (d *daemon) CreateHost(
if joiningBootstrap.Secrets, err = secrets.Export(
ctx, d.secretsStore, []secrets.ID{
garageRPCSecretSecretID,
garageS3APIGlobalBucketCredentialsSecretID,
},
); err != nil {
return JoiningBootstrap{}, fmt.Errorf("exporting secrets: %w", err)

7
go/daemon/garage_client_params.go
View File

@ -24,6 +24,11 @@ func (d *daemon) getGarageClientParams(
) (
GarageClientParams, error,
) {
creds, err := getGarageS3APIGlobalBucketCredentials(ctx, d.secretsStore)
if err != nil {
return GarageClientParams{}, fmt.Errorf("getting garage global bucket creds: %w", err)
}
rpcSecret, err := getGarageRPCSecret(ctx, d.secretsStore)
if err != nil && !errors.Is(err, secrets.ErrNotFound) {
return GarageClientParams{}, fmt.Errorf("getting garage rpc secret: %w", err)
@ -31,7 +36,7 @@ func (d *daemon) getGarageClientParams(
return GarageClientParams{
Peer: currBootstrap.ChooseGaragePeer(),
GlobalBucketS3APICredentials: currBootstrap.Garage.GlobalBucketS3APICredentials,
GlobalBucketS3APICredentials: creds,
RPCSecret: rpcSecret,
}, nil
}

5
go/daemon/global_bucket.go
View File

@ -24,12 +24,13 @@ func garageInitializeGlobalBucket(
ctx context.Context,
logger *mlog.Logger,
daemonConfig Config,
adminToken string,
hostBootstrap bootstrap.Bootstrap,
) (
garage.S3APICredentials, error,
) {
adminClient := NewGarageAdminClient(
logger, daemonConfig, hostBootstrap,
adminClient := newGarageAdminClient(
logger, daemonConfig, adminToken, hostBootstrap,
)
creds, err := adminClient.CreateS3APICredentials(