diff --git a/go/admin/admin.go b/go/admin/admin.go index 1e3bb79..aa39a1c 100644 --- a/go/admin/admin.go +++ b/go/admin/admin.go @@ -4,7 +4,6 @@ package admin import ( "encoding/json" "io" - "isle/garage" "isle/nebula" ) @@ -23,10 +22,6 @@ type Admin struct { Nebula struct { CACredentials nebula.CACredentials } - - Garage struct { - GlobalBucketS3APICredentials garage.S3APICredentials - } } // FromReader reads an admin.json from the given io.Reader. diff --git a/go/bootstrap/bootstrap.go b/go/bootstrap/bootstrap.go index 2ea9078..d51452a 100644 --- a/go/bootstrap/bootstrap.go +++ b/go/bootstrap/bootstrap.go @@ -7,7 +7,6 @@ import ( "encoding/json" "fmt" "isle/admin" - "isle/garage" "isle/nebula" "net/netip" "path/filepath" @@ -26,20 +25,11 @@ func AppDirPath(appDirPath string) string { return filepath.Join(appDirPath, "share/bootstrap.json") } -// Garage contains parameters needed to connect to and use the garage cluster. -type Garage struct { - AdminToken string - - // TODO this should be part of admin.CreationParams - GlobalBucketS3APICredentials garage.S3APICredentials -} - // Bootstrap contains all information which is needed by a host daemon to join a // network on boot. type Bootstrap struct { AdminCreationParams admin.CreationParams CAPublicCredentials nebula.CAPublicCredentials - Garage Garage PrivateCredentials nebula.HostPrivateCredentials HostAssigned `json:"-"` @@ -53,7 +43,6 @@ type Bootstrap struct { func New( caCreds nebula.CACredentials, adminCreationParams admin.CreationParams, - garage Garage, name nebula.HostName, ip netip.Addr, ) ( @@ -79,7 +68,6 @@ func New( return Bootstrap{ AdminCreationParams: adminCreationParams, CAPublicCredentials: caCreds.Public, - Garage: garage, PrivateCredentials: hostPrivCreds, HostAssigned: assigned, SignedHostAssigned: signedAssigned, diff --git a/go/daemon/child_garage.go b/go/daemon/child_garage.go index 43da3a0..4249548 100644 --- a/go/daemon/child_garage.go +++ b/go/daemon/child_garage.go @@ -19,11 +19,12 @@ func garageAdminClientLogger(logger *mlog.Logger) *mlog.Logger { return logger.WithNamespace("garageAdminClient") } -// NewGarageAdminClient will return an AdminClient for a local garage instance, +// newGarageAdminClient will return an AdminClient for a local garage instance, // or it will _panic_ if there is no local instance configured. -func NewGarageAdminClient( +func newGarageAdminClient( logger *mlog.Logger, daemonConfig Config, + adminToken string, hostBootstrap bootstrap.Bootstrap, ) *garage.AdminClient { @@ -35,7 +36,7 @@ func NewGarageAdminClient( thisHost.IP().String(), strconv.Itoa(daemonConfig.Storage.Allocations[0].AdminPort), ), - hostBootstrap.Garage.AdminToken, + adminToken, ) } @@ -43,6 +44,7 @@ func waitForGarage( ctx context.Context, logger *mlog.Logger, daemonConfig Config, + adminToken string, hostBootstrap bootstrap.Bootstrap, ) error { @@ -64,9 +66,7 @@ func waitForGarage( ) adminClient := garage.NewAdminClient( - adminClientLogger, - adminAddr, - hostBootstrap.Garage.AdminToken, + adminClientLogger, adminAddr, adminToken, ) ctx := mctx.Annotate(ctx, "garageAdminAddr", adminAddr) @@ -101,7 +101,7 @@ func bootstrapGarageHostForAlloc( } func garageWriteChildConfig( - rpcSecret, runtimeDirPath string, + rpcSecret, runtimeDirPath, adminToken string, hostBootstrap bootstrap.Bootstrap, alloc ConfigStorageAllocation, ) ( @@ -130,7 +130,7 @@ func garageWriteChildConfig( DataPath: alloc.DataPath, RPCSecret: rpcSecret, - AdminToken: hostBootstrap.Garage.AdminToken, + AdminToken: adminToken, LocalPeer: peer, BootstrapPeers: hostBootstrap.GaragePeers(), @@ -148,6 +148,7 @@ func garagePmuxProcConfigs( logger *mlog.Logger, rpcSecret, runtimeDirPath, binDirPath string, daemonConfig Config, + adminToken string, hostBootstrap bootstrap.Bootstrap, ) ( []pmuxlib.ProcessConfig, error, @@ -165,7 +166,7 @@ func garagePmuxProcConfigs( for _, alloc := range allocs { childConfigPath, err := garageWriteChildConfig( - rpcSecret, runtimeDirPath, hostBootstrap, alloc, + rpcSecret, runtimeDirPath, adminToken, hostBootstrap, alloc, ) if err != nil { return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err) @@ -188,15 +189,18 @@ func garageApplyLayout( ctx context.Context, logger *mlog.Logger, daemonConfig Config, + adminToken string, hostBootstrap bootstrap.Bootstrap, ) error { var ( - adminClient = NewGarageAdminClient(logger, daemonConfig, hostBootstrap) - thisHost = hostBootstrap.ThisHost() - hostName = thisHost.Name - allocs = daemonConfig.Storage.Allocations - peers = make([]garage.PeerLayout, len(allocs)) + adminClient = newGarageAdminClient( + logger, daemonConfig, adminToken, hostBootstrap, + ) + thisHost = hostBootstrap.ThisHost() + hostName = thisHost.Name + allocs = daemonConfig.Storage.Allocations + peers = make([]garage.PeerLayout, len(allocs)) ) for i, alloc := range allocs { diff --git a/go/daemon/child_pmux.go b/go/daemon/child_pmux.go index 117017f..34e0247 100644 --- a/go/daemon/child_pmux.go +++ b/go/daemon/child_pmux.go @@ -12,6 +12,7 @@ func (c *Children) newPmuxConfig( ctx context.Context, garageRPCSecret, binDirPath string, daemonConfig Config, + garageAdminToken string, hostBootstrap bootstrap.Bootstrap, ) ( pmuxlib.Config, error, @@ -45,6 +46,7 @@ func (c *Children) newPmuxConfig( c.opts.EnvVars.RuntimeDirPath, binDirPath, daemonConfig, + garageAdminToken, hostBootstrap, ) if err != nil { @@ -67,6 +69,7 @@ func (c *Children) newPmuxConfig( func (c *Children) postPmuxInit( ctx context.Context, daemonConfig Config, + garageAdminToken string, hostBootstrap bootstrap.Bootstrap, ) error { c.logger.Info(ctx, "Waiting for nebula VPN to come online") @@ -75,7 +78,9 @@ func (c *Children) postPmuxInit( } c.logger.Info(ctx, "Waiting for garage instances to come online") - err := waitForGarage(ctx, c.logger, daemonConfig, hostBootstrap) + err := waitForGarage( + ctx, c.logger, daemonConfig, garageAdminToken, hostBootstrap, + ) if err != nil { return fmt.Errorf("waiting for garage to start: %w", err) } diff --git a/go/daemon/children.go b/go/daemon/children.go index 74b0554..eac8718 100644 --- a/go/daemon/children.go +++ b/go/daemon/children.go @@ -32,6 +32,7 @@ func NewChildren( binDirPath string, secretsStore secrets.Store, daemonConfig Config, + garageAdminToken string, hostBootstrap bootstrap.Bootstrap, opts *Opts, ) ( @@ -55,7 +56,12 @@ func NewChildren( } pmuxConfig, err := c.newPmuxConfig( - ctx, garageRPCSecret, binDirPath, daemonConfig, hostBootstrap, + ctx, + garageRPCSecret, + binDirPath, + daemonConfig, + garageAdminToken, + hostBootstrap, ) if err != nil { return nil, fmt.Errorf("generating pmux config: %w", err) @@ -67,7 +73,9 @@ func NewChildren( c.logger.Debug(pmuxCtx, "pmux stopped") }() - initErr := c.postPmuxInit(ctx, daemonConfig, hostBootstrap) + initErr := c.postPmuxInit( + ctx, daemonConfig, garageAdminToken, hostBootstrap, + ) if initErr != nil { logger.Warn(ctx, "failed to initialize Children, shutting down child processes", err) if err := c.Shutdown(); err != nil { diff --git a/go/daemon/daemon.go b/go/daemon/daemon.go index 8ac7115..10231c7 100644 --- a/go/daemon/daemon.go +++ b/go/daemon/daemon.go @@ -11,7 +11,6 @@ import ( "io/fs" "isle/admin" "isle/bootstrap" - "isle/garage" "isle/jsonutil" "isle/nebula" "isle/secrets" @@ -141,7 +140,8 @@ type daemon struct { envBinDirPath string opts *Opts - secretsStore secrets.Store + secretsStore secrets.Store + garageAdminToken string l sync.RWMutex state int @@ -177,11 +177,12 @@ func NewDaemon( ) { var ( d = &daemon{ - logger: logger, - daemonConfig: daemonConfig, - envBinDirPath: envBinDirPath, - opts: opts.withDefaults(), - shutdownCh: make(chan struct{}), + logger: logger, + daemonConfig: daemonConfig, + envBinDirPath: envBinDirPath, + opts: opts.withDefaults(), + garageAdminToken: randStr(32), + shutdownCh: make(chan struct{}), } bootstrapFilePath = bootstrap.StateDirPath(d.opts.EnvVars.StateDirPath) ) @@ -370,7 +371,11 @@ func (d *daemon) postInit(ctx context.Context) bool { "Applying garage layout", func(ctx context.Context) error { return garageApplyLayout( - ctx, d.logger, d.daemonConfig, d.currBootstrap, + ctx, + d.logger, + d.daemonConfig, + d.garageAdminToken, + d.currBootstrap, ) }, ) { @@ -383,28 +388,29 @@ func (d *daemon) postInit(ctx context.Context) bool { // // TODO this is pretty hacky, but there doesn't seem to be a better way to // manage it at the moment. - if d.currBootstrap.Garage.GlobalBucketS3APICredentials == (garage.S3APICredentials{}) { - currBootstrap := d.currBootstrap + _, err := getGarageS3APIGlobalBucketCredentials(ctx, d.secretsStore) + if errors.Is(err, secrets.ErrNotFound) { if !until( ctx, d.logger, "Initializing garage shared global bucket", func(ctx context.Context) error { garageGlobalBucketCreds, err := garageInitializeGlobalBucket( - ctx, d.logger, d.daemonConfig, d.currBootstrap, + ctx, + d.logger, + d.daemonConfig, + d.garageAdminToken, + d.currBootstrap, ) if err != nil { return fmt.Errorf("initializing global bucket: %w", err) } - currBootstrap.Garage.GlobalBucketS3APICredentials = garageGlobalBucketCreds - - d.logger.Info(ctx, "Writing bootstrap to state directory") - err = writeBootstrapToStateDir( - d.opts.EnvVars.StateDirPath, currBootstrap, + err = setGarageS3APIGlobalBucketCredentials( + ctx, d.secretsStore, garageGlobalBucketCreds, ) if err != nil { - return fmt.Errorf("writing bootstrap to state dir: %w", err) + return fmt.Errorf("storing global bucket creds: %w", err) } return nil @@ -412,10 +418,6 @@ func (d *daemon) postInit(ctx context.Context) bool { ) { return false } - - d.l.Lock() - d.currBootstrap = currBootstrap - d.l.Unlock() } if !until( @@ -461,6 +463,7 @@ func (d *daemon) restartLoop(ctx context.Context, readyCh chan<- struct{}) { d.envBinDirPath, d.secretsStore, d.daemonConfig, + d.garageAdminToken, d.currBootstrap, d.opts, ) @@ -533,13 +536,9 @@ func (d *daemon) CreateNetwork( }, } - garageBootstrap = bootstrap.Garage{ - AdminToken: randStr(32), - } + garageRPCSecret = randStr(32) ) - garageRPCSecret := randStr(32) - err = setGarageRPCSecret(ctx, d.secretsStore, garageRPCSecret) if err != nil { return admin.Admin{}, fmt.Errorf("storing garage RPC secret: %w", err) @@ -548,7 +547,6 @@ func (d *daemon) CreateNetwork( hostBootstrap, err := bootstrap.New( nebulaCACreds, adm.CreationParams, - garageBootstrap, hostName, ipNet.FirstAddr(), ) @@ -584,16 +582,7 @@ func (d *daemon) CreateNetwork( return adm, ctx.Err() } - // As part of postInit, which is called prior to ready(), the restartLoop - // will check if the global bucket creds have been created yet or not, and - // create them if so. So once ready() is called we can get the created creds - // from the currBootstrap - d.l.RLock() - garageGlobalBucketCreds := d.currBootstrap.Garage.GlobalBucketS3APICredentials - d.l.RUnlock() - adm.Nebula.CACredentials = nebulaCACreds - adm.Garage.GlobalBucketS3APICredentials = garageGlobalBucketCreds return adm, nil } @@ -697,13 +686,6 @@ func (d *daemon) CreateHost( JoiningBootstrap, error, ) { var ( - garageGlobalBucketS3APICreds = currBootstrap.Garage.GlobalBucketS3APICredentials - - garageBootstrap = bootstrap.Garage{ - AdminToken: randStr(32), - GlobalBucketS3APICredentials: garageGlobalBucketS3APICreds, - } - joiningBootstrap JoiningBootstrap err error ) @@ -711,7 +693,6 @@ func (d *daemon) CreateHost( joiningBootstrap.Bootstrap, err = bootstrap.New( makeCACreds(currBootstrap, caSigningPrivateKey), currBootstrap.AdminCreationParams, - garageBootstrap, hostName, ip, ) @@ -726,6 +707,7 @@ func (d *daemon) CreateHost( if joiningBootstrap.Secrets, err = secrets.Export( ctx, d.secretsStore, []secrets.ID{ garageRPCSecretSecretID, + garageS3APIGlobalBucketCredentialsSecretID, }, ); err != nil { return JoiningBootstrap{}, fmt.Errorf("exporting secrets: %w", err) diff --git a/go/daemon/garage_client_params.go b/go/daemon/garage_client_params.go index fd16a9a..2474241 100644 --- a/go/daemon/garage_client_params.go +++ b/go/daemon/garage_client_params.go @@ -24,6 +24,11 @@ func (d *daemon) getGarageClientParams( ) ( GarageClientParams, error, ) { + creds, err := getGarageS3APIGlobalBucketCredentials(ctx, d.secretsStore) + if err != nil { + return GarageClientParams{}, fmt.Errorf("getting garage global bucket creds: %w", err) + } + rpcSecret, err := getGarageRPCSecret(ctx, d.secretsStore) if err != nil && !errors.Is(err, secrets.ErrNotFound) { return GarageClientParams{}, fmt.Errorf("getting garage rpc secret: %w", err) @@ -31,7 +36,7 @@ func (d *daemon) getGarageClientParams( return GarageClientParams{ Peer: currBootstrap.ChooseGaragePeer(), - GlobalBucketS3APICredentials: currBootstrap.Garage.GlobalBucketS3APICredentials, + GlobalBucketS3APICredentials: creds, RPCSecret: rpcSecret, }, nil } diff --git a/go/daemon/global_bucket.go b/go/daemon/global_bucket.go index eeb96bb..d12f413 100644 --- a/go/daemon/global_bucket.go +++ b/go/daemon/global_bucket.go @@ -24,12 +24,13 @@ func garageInitializeGlobalBucket( ctx context.Context, logger *mlog.Logger, daemonConfig Config, + adminToken string, hostBootstrap bootstrap.Bootstrap, ) ( garage.S3APICredentials, error, ) { - adminClient := NewGarageAdminClient( - logger, daemonConfig, hostBootstrap, + adminClient := newGarageAdminClient( + logger, daemonConfig, adminToken, hostBootstrap, ) creds, err := adminClient.CreateS3APICredentials(