Use secrets store for global garage bucket creds

go/admin/admin.go

@@ -4,7 +4,6 @@ package admin
 import (
 	"encoding/json"
 	"io"
-	"isle/garage"
 	"isle/nebula"
 )
 
@@ -23,10 +22,6 @@ type Admin struct {
 	Nebula struct {
 		CACredentials nebula.CACredentials
 	}
-
-	Garage struct {
-		GlobalBucketS3APICredentials garage.S3APICredentials
-	}
 }
 
 // FromReader reads an admin.json from the given io.Reader.

go/bootstrap/bootstrap.go

@@ -7,7 +7,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"isle/admin"
-	"isle/garage"
 	"isle/nebula"
 	"net/netip"
 	"path/filepath"
@@ -26,20 +25,11 @@ func AppDirPath(appDirPath string) string {
 	return filepath.Join(appDirPath, "share/bootstrap.json")
 }
 
-// Garage contains parameters needed to connect to and use the garage cluster.
-type Garage struct {
-	AdminToken string
-
-	// TODO this should be part of admin.CreationParams
-	GlobalBucketS3APICredentials garage.S3APICredentials
-}
-
 // Bootstrap contains all information which is needed by a host daemon to join a
 // network on boot.
 type Bootstrap struct {
 	AdminCreationParams admin.CreationParams
 	CAPublicCredentials nebula.CAPublicCredentials
-	Garage              Garage
 
 	PrivateCredentials nebula.HostPrivateCredentials
 	HostAssigned       `json:"-"`
@@ -53,7 +43,6 @@ type Bootstrap struct {
 func New(
 	caCreds nebula.CACredentials,
 	adminCreationParams admin.CreationParams,
-	garage Garage,
 	name nebula.HostName,
 	ip netip.Addr,
 ) (
@@ -79,7 +68,6 @@ func New(
 	return Bootstrap{
 		AdminCreationParams: adminCreationParams,
 		CAPublicCredentials: caCreds.Public,
-		Garage:              garage,
 		PrivateCredentials:  hostPrivCreds,
 		HostAssigned:        assigned,
 		SignedHostAssigned:  signedAssigned,

go/daemon/child_garage.go

@@ -19,11 +19,12 @@ func garageAdminClientLogger(logger *mlog.Logger) *mlog.Logger {
 	return logger.WithNamespace("garageAdminClient")
 }
 
-// NewGarageAdminClient will return an AdminClient for a local garage instance,
+// newGarageAdminClient will return an AdminClient for a local garage instance,
 // or it will _panic_ if there is no local instance configured.
-func NewGarageAdminClient(
+func newGarageAdminClient(
 	logger *mlog.Logger,
 	daemonConfig Config,
+	adminToken string,
 	hostBootstrap bootstrap.Bootstrap,
 ) *garage.AdminClient {
 
@@ -35,7 +36,7 @@ func NewGarageAdminClient(
 			thisHost.IP().String(),
 			strconv.Itoa(daemonConfig.Storage.Allocations[0].AdminPort),
 		),
-		hostBootstrap.Garage.AdminToken,
+		adminToken,
 	)
 }
 
@@ -43,6 +44,7 @@ func waitForGarage(
 	ctx context.Context,
 	logger *mlog.Logger,
 	daemonConfig Config,
+	adminToken string,
 	hostBootstrap bootstrap.Bootstrap,
 ) error {
 
@@ -64,9 +66,7 @@ func waitForGarage(
 		)
 
 		adminClient := garage.NewAdminClient(
-			adminClientLogger,
+			adminClientLogger, adminAddr, adminToken,
-			adminAddr,
-			hostBootstrap.Garage.AdminToken,
 		)
 
 		ctx := mctx.Annotate(ctx, "garageAdminAddr", adminAddr)
@@ -101,7 +101,7 @@ func bootstrapGarageHostForAlloc(
 }
 
 func garageWriteChildConfig(
-	rpcSecret, runtimeDirPath string,
+	rpcSecret, runtimeDirPath, adminToken string,
 	hostBootstrap bootstrap.Bootstrap,
 	alloc ConfigStorageAllocation,
 ) (
@@ -130,7 +130,7 @@ func garageWriteChildConfig(
 		DataPath: alloc.DataPath,
 
 		RPCSecret:  rpcSecret,
-		AdminToken: hostBootstrap.Garage.AdminToken,
+		AdminToken: adminToken,
 
 		LocalPeer:      peer,
 		BootstrapPeers: hostBootstrap.GaragePeers(),
@@ -148,6 +148,7 @@ func garagePmuxProcConfigs(
 	logger *mlog.Logger,
 	rpcSecret, runtimeDirPath, binDirPath string,
 	daemonConfig Config,
+	adminToken string,
 	hostBootstrap bootstrap.Bootstrap,
 ) (
 	[]pmuxlib.ProcessConfig, error,
@@ -165,7 +166,7 @@ func garagePmuxProcConfigs(
 	for _, alloc := range allocs {
 
 		childConfigPath, err := garageWriteChildConfig(
-			rpcSecret, runtimeDirPath, hostBootstrap, alloc,
+			rpcSecret, runtimeDirPath, adminToken, hostBootstrap, alloc,
 		)
 		if err != nil {
 			return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
@@ -188,15 +189,18 @@ func garageApplyLayout(
 	ctx context.Context,
 	logger *mlog.Logger,
 	daemonConfig Config,
+	adminToken string,
 	hostBootstrap bootstrap.Bootstrap,
 ) error {
 
 	var (
-		adminClient = NewGarageAdminClient(logger, daemonConfig, hostBootstrap)
+		adminClient = newGarageAdminClient(
-		thisHost    = hostBootstrap.ThisHost()
+			logger, daemonConfig, adminToken, hostBootstrap,
-		hostName    = thisHost.Name
+		)
-		allocs      = daemonConfig.Storage.Allocations
+		thisHost = hostBootstrap.ThisHost()
-		peers       = make([]garage.PeerLayout, len(allocs))
+		hostName = thisHost.Name
+		allocs   = daemonConfig.Storage.Allocations
+		peers    = make([]garage.PeerLayout, len(allocs))
 	)
 
 	for i, alloc := range allocs {

go/daemon/child_pmux.go

@@ -12,6 +12,7 @@ func (c *Children) newPmuxConfig(
 	ctx context.Context,
 	garageRPCSecret, binDirPath string,
 	daemonConfig Config,
+	garageAdminToken string,
 	hostBootstrap bootstrap.Bootstrap,
 ) (
 	pmuxlib.Config, error,
@@ -45,6 +46,7 @@ func (c *Children) newPmuxConfig(
 		c.opts.EnvVars.RuntimeDirPath,
 		binDirPath,
 		daemonConfig,
+		garageAdminToken,
 		hostBootstrap,
 	)
 	if err != nil {
@@ -67,6 +69,7 @@ func (c *Children) newPmuxConfig(
 func (c *Children) postPmuxInit(
 	ctx context.Context,
 	daemonConfig Config,
+	garageAdminToken string,
 	hostBootstrap bootstrap.Bootstrap,
 ) error {
 	c.logger.Info(ctx, "Waiting for nebula VPN to come online")
@@ -75,7 +78,9 @@ func (c *Children) postPmuxInit(
 	}
 
 	c.logger.Info(ctx, "Waiting for garage instances to come online")
-	err := waitForGarage(ctx, c.logger, daemonConfig, hostBootstrap)
+	err := waitForGarage(
+		ctx, c.logger, daemonConfig, garageAdminToken, hostBootstrap,
+	)
 	if err != nil {
 		return fmt.Errorf("waiting for garage to start: %w", err)
 	}

go/daemon/children.go

@@ -32,6 +32,7 @@ func NewChildren(
 	binDirPath string,
 	secretsStore secrets.Store,
 	daemonConfig Config,
+	garageAdminToken string,
 	hostBootstrap bootstrap.Bootstrap,
 	opts *Opts,
 ) (
@@ -55,7 +56,12 @@ func NewChildren(
 	}
 
 	pmuxConfig, err := c.newPmuxConfig(
-		ctx, garageRPCSecret, binDirPath, daemonConfig, hostBootstrap,
+		ctx,
+		garageRPCSecret,
+		binDirPath,
+		daemonConfig,
+		garageAdminToken,
+		hostBootstrap,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("generating pmux config: %w", err)
@@ -67,7 +73,9 @@ func NewChildren(
 		c.logger.Debug(pmuxCtx, "pmux stopped")
 	}()
 
-	initErr := c.postPmuxInit(ctx, daemonConfig, hostBootstrap)
+	initErr := c.postPmuxInit(
+		ctx, daemonConfig, garageAdminToken, hostBootstrap,
+	)
 	if initErr != nil {
 		logger.Warn(ctx, "failed to initialize Children, shutting down child processes", err)
 		if err := c.Shutdown(); err != nil {

go/daemon/daemon.go

@@ -11,7 +11,6 @@ import (
 	"io/fs"
 	"isle/admin"
 	"isle/bootstrap"
-	"isle/garage"
 	"isle/jsonutil"
 	"isle/nebula"
 	"isle/secrets"
@@ -141,7 +140,8 @@ type daemon struct {
 	envBinDirPath string
 	opts          *Opts
 
-	secretsStore secrets.Store
+	secretsStore     secrets.Store
+	garageAdminToken string
 
 	l             sync.RWMutex
 	state         int
@@ -177,11 +177,12 @@ func NewDaemon(
 ) {
 	var (
 		d = &daemon{
-			logger:        logger,
+			logger:           logger,
-			daemonConfig:  daemonConfig,
+			daemonConfig:     daemonConfig,
-			envBinDirPath: envBinDirPath,
+			envBinDirPath:    envBinDirPath,
-			opts:          opts.withDefaults(),
+			opts:             opts.withDefaults(),
-			shutdownCh:    make(chan struct{}),
+			garageAdminToken: randStr(32),
+			shutdownCh:       make(chan struct{}),
 		}
 		bootstrapFilePath = bootstrap.StateDirPath(d.opts.EnvVars.StateDirPath)
 	)
@@ -370,7 +371,11 @@ func (d *daemon) postInit(ctx context.Context) bool {
 			"Applying garage layout",
 			func(ctx context.Context) error {
 				return garageApplyLayout(
-					ctx, d.logger, d.daemonConfig, d.currBootstrap,
+					ctx,
+					d.logger,
+					d.daemonConfig,
+					d.garageAdminToken,
+					d.currBootstrap,
 				)
 			},
 		) {
@@ -383,28 +388,29 @@ func (d *daemon) postInit(ctx context.Context) bool {
 	//
 	// TODO this is pretty hacky, but there doesn't seem to be a better way to
 	// manage it at the moment.
-	if d.currBootstrap.Garage.GlobalBucketS3APICredentials == (garage.S3APICredentials{}) {
+	_, err := getGarageS3APIGlobalBucketCredentials(ctx, d.secretsStore)
-		currBootstrap := d.currBootstrap
+	if errors.Is(err, secrets.ErrNotFound) {
 		if !until(
 			ctx,
 			d.logger,
 			"Initializing garage shared global bucket",
 			func(ctx context.Context) error {
 				garageGlobalBucketCreds, err := garageInitializeGlobalBucket(
-					ctx, d.logger, d.daemonConfig, d.currBootstrap,
+					ctx,
+					d.logger,
+					d.daemonConfig,
+					d.garageAdminToken,
+					d.currBootstrap,
 				)
 				if err != nil {
 					return fmt.Errorf("initializing global bucket: %w", err)
 				}
 
-				currBootstrap.Garage.GlobalBucketS3APICredentials = garageGlobalBucketCreds
+				err = setGarageS3APIGlobalBucketCredentials(
-
+					ctx, d.secretsStore, garageGlobalBucketCreds,
-				d.logger.Info(ctx, "Writing bootstrap to state directory")
-				err = writeBootstrapToStateDir(
-					d.opts.EnvVars.StateDirPath, currBootstrap,
 				)
 				if err != nil {
-					return fmt.Errorf("writing bootstrap to state dir: %w", err)
+					return fmt.Errorf("storing global bucket creds: %w", err)
 				}
 
 				return nil
@@ -412,10 +418,6 @@ func (d *daemon) postInit(ctx context.Context) bool {
 		) {
 			return false
 		}
-
-		d.l.Lock()
-		d.currBootstrap = currBootstrap
-		d.l.Unlock()
 	}
 
 	if !until(
@@ -461,6 +463,7 @@ func (d *daemon) restartLoop(ctx context.Context, readyCh chan<- struct{}) {
 			d.envBinDirPath,
 			d.secretsStore,
 			d.daemonConfig,
+			d.garageAdminToken,
 			d.currBootstrap,
 			d.opts,
 		)
@@ -533,13 +536,9 @@ func (d *daemon) CreateNetwork(
 			},
 		}
 
-		garageBootstrap = bootstrap.Garage{
+		garageRPCSecret = randStr(32)
-			AdminToken: randStr(32),
-		}
 	)
 
-	garageRPCSecret := randStr(32)
-
 	err = setGarageRPCSecret(ctx, d.secretsStore, garageRPCSecret)
 	if err != nil {
 		return admin.Admin{}, fmt.Errorf("storing garage RPC secret: %w", err)
@@ -548,7 +547,6 @@ func (d *daemon) CreateNetwork(
 	hostBootstrap, err := bootstrap.New(
 		nebulaCACreds,
 		adm.CreationParams,
-		garageBootstrap,
 		hostName,
 		ipNet.FirstAddr(),
 	)
@@ -584,16 +582,7 @@ func (d *daemon) CreateNetwork(
 		return adm, ctx.Err()
 	}
 
-	// As part of postInit, which is called prior to ready(), the restartLoop
-	// will check if the global bucket creds have been created yet or not, and
-	// create them if so. So once ready() is called we can get the created creds
-	// from the currBootstrap
-	d.l.RLock()
-	garageGlobalBucketCreds := d.currBootstrap.Garage.GlobalBucketS3APICredentials
-	d.l.RUnlock()
-
 	adm.Nebula.CACredentials = nebulaCACreds
-	adm.Garage.GlobalBucketS3APICredentials = garageGlobalBucketCreds
 
 	return adm, nil
 }
@@ -697,13 +686,6 @@ func (d *daemon) CreateHost(
 		JoiningBootstrap, error,
 	) {
 		var (
-			garageGlobalBucketS3APICreds = currBootstrap.Garage.GlobalBucketS3APICredentials
-
-			garageBootstrap = bootstrap.Garage{
-				AdminToken:                   randStr(32),
-				GlobalBucketS3APICredentials: garageGlobalBucketS3APICreds,
-			}
-
 			joiningBootstrap JoiningBootstrap
 			err              error
 		)
@@ -711,7 +693,6 @@ func (d *daemon) CreateHost(
 		joiningBootstrap.Bootstrap, err = bootstrap.New(
 			makeCACreds(currBootstrap, caSigningPrivateKey),
 			currBootstrap.AdminCreationParams,
-			garageBootstrap,
 			hostName,
 			ip,
 		)
@@ -726,6 +707,7 @@ func (d *daemon) CreateHost(
 		if joiningBootstrap.Secrets, err = secrets.Export(
 			ctx, d.secretsStore, []secrets.ID{
 				garageRPCSecretSecretID,
+				garageS3APIGlobalBucketCredentialsSecretID,
 			},
 		); err != nil {
 			return JoiningBootstrap{}, fmt.Errorf("exporting secrets: %w", err)

go/daemon/garage_client_params.go

@@ -24,6 +24,11 @@ func (d *daemon) getGarageClientParams(
 ) (
 	GarageClientParams, error,
 ) {
+	creds, err := getGarageS3APIGlobalBucketCredentials(ctx, d.secretsStore)
+	if err != nil {
+		return GarageClientParams{}, fmt.Errorf("getting garage global bucket creds: %w", err)
+	}
+
 	rpcSecret, err := getGarageRPCSecret(ctx, d.secretsStore)
 	if err != nil && !errors.Is(err, secrets.ErrNotFound) {
 		return GarageClientParams{}, fmt.Errorf("getting garage rpc secret: %w", err)
@@ -31,7 +36,7 @@ func (d *daemon) getGarageClientParams(
 
 	return GarageClientParams{
 		Peer:                         currBootstrap.ChooseGaragePeer(),
-		GlobalBucketS3APICredentials: currBootstrap.Garage.GlobalBucketS3APICredentials,
+		GlobalBucketS3APICredentials: creds,
 		RPCSecret:                    rpcSecret,
 	}, nil
 }

go/daemon/global_bucket.go

@@ -24,12 +24,13 @@ func garageInitializeGlobalBucket(
 	ctx context.Context,
 	logger *mlog.Logger,
 	daemonConfig Config,
+	adminToken string,
 	hostBootstrap bootstrap.Bootstrap,
 ) (
 	garage.S3APICredentials, error,
 ) {
-	adminClient := NewGarageAdminClient(
+	adminClient := newGarageAdminClient(
-		logger, daemonConfig, hostBootstrap,
+		logger, daemonConfig, adminToken, hostBootstrap,
 	)
 
 	creds, err := adminClient.CreateS3APICredentials(