From 4e5d3b28ab85b701b39ffcc6aa838dcf4b171b57 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Mon, 17 Jun 2024 22:15:28 +0200 Subject: [PATCH] Move some Bootstrap methods onto Daemon --- go/bootstrap/garage_global_bucket.go | 53 ++---------- go/cmd/entrypoint/admin.go | 3 +- go/cmd/entrypoint/bootstrap_util.go | 2 +- go/cmd/entrypoint/daemon.go | 12 ++- go/cmd/entrypoint/daemon_util.go | 10 +-- go/daemon/daemon.go | 14 ++- go/daemon/global_bucket.go | 122 +++++++++++++++++++++++++++ 7 files changed, 155 insertions(+), 61 deletions(-) create mode 100644 go/daemon/global_bucket.go diff --git a/go/bootstrap/garage_global_bucket.go b/go/bootstrap/garage_global_bucket.go index 0248893..9207757 100644 --- a/go/bootstrap/garage_global_bucket.go +++ b/go/bootstrap/garage_global_bucket.go @@ -1,12 +1,10 @@ package bootstrap import ( - "bytes" "context" "encoding/json" "fmt" "isle/garage" - "isle/nebula" "path/filepath" "github.com/mediocregopher/mediocre-go-lib/v2/mctx" @@ -14,56 +12,13 @@ import ( "github.com/minio/minio-go/v7" ) -// Paths within garage's global bucket +// Paths within garage's global bucket. +// +// TODO this is getting moved into daemon package. const ( garageGlobalBucketBootstrapHostsDirPath = "bootstrap/hosts" ) -// PutGarageBoostrapHost places the .json.signed file for this host -// into garage so that other hosts are able to see relevant configuration for -// it. -func (b Bootstrap) PutGarageBoostrapHost(ctx context.Context) error { - var ( - host = b.ThisHost() - client = b.GlobalBucketS3APIClient() - ) - - configured, err := nebula.Sign( - host.HostConfigured, b.PrivateCredentials.SigningPrivateKey, - ) - if err != nil { - return fmt.Errorf("signing host configured data: %w", err) - } - - hostB, err := json.Marshal(AuthenticatedHost{ - Assigned: b.SignedHostAssigned, - Configured: configured, - }) - if err != nil { - return fmt.Errorf("encoding host data: %w", err) - } - - filePath := filepath.Join( - garageGlobalBucketBootstrapHostsDirPath, - host.Name+".json.signed", - ) - - _, err = client.PutObject( - ctx, - garage.GlobalBucket, - filePath, - bytes.NewReader(hostB), - int64(len(hostB)), - minio.PutObjectOptions{}, - ) - - if err != nil { - return fmt.Errorf("writing to %q in global bucket: %w", filePath, err) - } - - return nil -} - // RemoveGarageBootstrapHost removes the .json.signed for the given // host from garage. // @@ -85,6 +40,8 @@ func RemoveGarageBootstrapHost( // GetGarageBootstrapHosts loads the .json.signed file for all hosts // stored in garage. +// +// Deprecated: should use the method off Daemon instead. func (b Bootstrap) GetGarageBootstrapHosts( ctx context.Context, logger *mlog.Logger, diff --git a/go/cmd/entrypoint/admin.go b/go/cmd/entrypoint/admin.go index e1ff7eb..b0c9e09 100644 --- a/go/cmd/entrypoint/admin.go +++ b/go/cmd/entrypoint/admin.go @@ -173,6 +173,7 @@ var subCmdAdminCreateNetwork = subCmd{ hostBootstrap, envRuntimeDirPath, envBinDirPath, + envStateDirPath, &daemon.Opts{ // SkipHostBootstrapPush is required, because the global bucket // hasn't actually been initialized yet, so there's nowhere to @@ -208,7 +209,7 @@ var subCmdAdminCreateNetwork = subCmd{ // rewrite the bootstrap now that the global bucket creds have been // added to it. - if err := writeBootstrapToDataDir(hostBootstrap); err != nil { + if err := writeBootstrapToStateDir(hostBootstrap); err != nil { return fmt.Errorf("writing bootstrap file: %w", err) } diff --git a/go/cmd/entrypoint/bootstrap_util.go b/go/cmd/entrypoint/bootstrap_util.go index 38ac306..0307214 100644 --- a/go/cmd/entrypoint/bootstrap_util.go +++ b/go/cmd/entrypoint/bootstrap_util.go @@ -27,7 +27,7 @@ func loadHostBootstrap() (bootstrap.Bootstrap, error) { return hostBootstrap, nil } -func writeBootstrapToDataDir(hostBootstrap bootstrap.Bootstrap) error { +func writeBootstrapToStateDir(hostBootstrap bootstrap.Bootstrap) error { path := bootstrap.StateDirPath(envStateDirPath) dirPath := filepath.Dir(path) diff --git a/go/cmd/entrypoint/daemon.go b/go/cmd/entrypoint/daemon.go index 3abc490..68bd62d 100644 --- a/go/cmd/entrypoint/daemon.go +++ b/go/cmd/entrypoint/daemon.go @@ -41,6 +41,7 @@ import ( func reloadBootstrap( ctx context.Context, logger *mlog.Logger, + daemonInst daemon.Daemon, hostBootstrap bootstrap.Bootstrap, ) ( bootstrap.Bootstrap, bool, error, @@ -48,7 +49,7 @@ func reloadBootstrap( thisHost := hostBootstrap.ThisHost() - newHosts, err := hostBootstrap.GetGarageBootstrapHosts(ctx, logger) + newHosts, err := daemonInst.GetGarageBootstrapHosts(ctx) if err != nil { return bootstrap.Bootstrap{}, false, fmt.Errorf("getting hosts from garage: %w", err) } @@ -73,7 +74,7 @@ func reloadBootstrap( hostBootstrap.Hosts = newHosts - if err := writeBootstrapToDataDir(hostBootstrap); err != nil { + if err := writeBootstrapToStateDir(hostBootstrap); err != nil { return bootstrap.Bootstrap{}, false, fmt.Errorf("writing new bootstrap to data dir: %w", err) } @@ -99,6 +100,7 @@ func runDaemonPmuxOnce( hostBootstrap, envRuntimeDirPath, envBinDirPath, + envStateDirPath, nil, ) if err != nil { @@ -131,7 +133,9 @@ func runDaemonPmuxOnce( err error ) - if hostBootstrap, changed, err = reloadBootstrap(ctx, logger, hostBootstrap); err != nil { + if hostBootstrap, changed, err = reloadBootstrap( + ctx, logger, daemonInst, hostBootstrap, + ); err != nil { return bootstrap.Bootstrap{}, fmt.Errorf("reloading bootstrap: %w", err) } else if changed { @@ -236,7 +240,7 @@ var subCmdDaemon = subCmd{ // If the bootstrap file is not being stored in the data dir, copy // it there, so it can be loaded from there next time. - if err := writeBootstrapToDataDir(hostBootstrap); err != nil { + if err := writeBootstrapToStateDir(hostBootstrap); err != nil { return fmt.Errorf("writing bootstrap.json to data dir: %w", err) } } diff --git a/go/cmd/entrypoint/daemon_util.go b/go/cmd/entrypoint/daemon_util.go index 17588a9..ea25076 100644 --- a/go/cmd/entrypoint/daemon_util.go +++ b/go/cmd/entrypoint/daemon_util.go @@ -8,12 +8,10 @@ import ( ) func coalesceDaemonConfigAndBootstrap( - hostBootstrap bootstrap.Bootstrap, - daemonConfig daemon.Config, + hostBootstrap bootstrap.Bootstrap, daemonConfig daemon.Config, ) ( bootstrap.Bootstrap, error, ) { - host := bootstrap.Host{ HostAssigned: hostBootstrap.HostAssigned, HostConfigured: bootstrap.HostConfigured{ @@ -29,7 +27,9 @@ func coalesceDaemonConfigAndBootstrap( id, rpcPort, err := garagesrv.InitAlloc(alloc.MetaPath, alloc.RPCPort) if err != nil { - return bootstrap.Bootstrap{}, fmt.Errorf("initializing alloc at %q: %w", alloc.MetaPath, err) + return bootstrap.Bootstrap{}, fmt.Errorf( + "initializing alloc at %q: %w", alloc.MetaPath, err, + ) } host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{ @@ -44,7 +44,7 @@ func coalesceDaemonConfigAndBootstrap( hostBootstrap.Hosts[host.Name] = host - if err := writeBootstrapToDataDir(hostBootstrap); err != nil { + if err := writeBootstrapToStateDir(hostBootstrap); err != nil { return bootstrap.Bootstrap{}, fmt.Errorf("writing bootstrap file: %w", err) } diff --git a/go/daemon/daemon.go b/go/daemon/daemon.go index 2610f6c..867a75c 100644 --- a/go/daemon/daemon.go +++ b/go/daemon/daemon.go @@ -18,6 +18,7 @@ type daemon struct { logger *mlog.Logger config Config hostBootstrap bootstrap.Bootstrap + stateDirPath string opts Opts pmuxCancelFn context.CancelFunc @@ -28,6 +29,14 @@ type daemon struct { // with isle, typically via the unix socket. type Daemon interface { + // GetGarageBootstrapHosts loads (and verifies) the .json.signed + // file for all hosts stored in garage. + GetGarageBootstrapHosts( + ctx context.Context, + ) ( + map[string]bootstrap.Host, error, + ) + // Shutdown blocks until all resources held or created by the daemon, // including child processes it has started, have been cleaned up, or until // the context is canceled. @@ -72,7 +81,7 @@ func New( logger *mlog.Logger, config Config, hostBootstrap bootstrap.Bootstrap, - runtimeDirPath, binDirPath string, + runtimeDirPath, binDirPath, stateDirPath string, opts *Opts, ) ( Daemon, error, @@ -116,6 +125,7 @@ func New( logger: logger, config: config, hostBootstrap: hostBootstrap, + stateDirPath: stateDirPath, opts: *opts, pmuxCancelFn: pmuxCancelFn, pmuxStoppedCh: make(chan struct{}), @@ -175,7 +185,7 @@ func (d *daemon) postPmuxInit(ctx context.Context) error { if !d.opts.SkipHostBootstrapPush { if err := until(ctx, func(ctx context.Context) error { - if err := d.hostBootstrap.PutGarageBoostrapHost(ctx); err != nil { + if err := d.putGarageBoostrapHost(ctx); err != nil { d.logger.Error(ctx, "updating host info in garage", err) return err } diff --git a/go/daemon/global_bucket.go b/go/daemon/global_bucket.go new file mode 100644 index 0000000..e1764f0 --- /dev/null +++ b/go/daemon/global_bucket.go @@ -0,0 +1,122 @@ +package daemon + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "isle/bootstrap" + "isle/garage" + "isle/nebula" + "path/filepath" + + "github.com/mediocregopher/mediocre-go-lib/v2/mctx" + "github.com/minio/minio-go/v7" +) + +// Paths within garage's global bucket. +const ( + garageGlobalBucketBootstrapHostsDirPath = "bootstrap/hosts" +) + +// putGarageBoostrapHost places the .json.signed file for this host +// into garage so that other hosts are able to see relevant configuration for +// it. +func (d *daemon) putGarageBoostrapHost(ctx context.Context) error { + var ( + b = d.hostBootstrap + host = b.ThisHost() + client = b.GlobalBucketS3APIClient() + ) + + configured, err := nebula.Sign( + host.HostConfigured, b.PrivateCredentials.SigningPrivateKey, + ) + if err != nil { + return fmt.Errorf("signing host configured data: %w", err) + } + + hostB, err := json.Marshal(bootstrap.AuthenticatedHost{ + Assigned: b.SignedHostAssigned, + Configured: configured, + }) + if err != nil { + return fmt.Errorf("encoding host data: %w", err) + } + + filePath := filepath.Join( + garageGlobalBucketBootstrapHostsDirPath, + host.Name+".json.signed", + ) + + _, err = client.PutObject( + ctx, + garage.GlobalBucket, + filePath, + bytes.NewReader(hostB), + int64(len(hostB)), + minio.PutObjectOptions{}, + ) + + if err != nil { + return fmt.Errorf("writing to %q in global bucket: %w", filePath, err) + } + + return nil +} + +func (d *daemon) GetGarageBootstrapHosts( + ctx context.Context, +) ( + map[string]bootstrap.Host, error, +) { + var ( + b = d.hostBootstrap + client = b.GlobalBucketS3APIClient() + hosts = map[string]bootstrap.Host{} + + objInfoCh = client.ListObjects( + ctx, garage.GlobalBucket, + minio.ListObjectsOptions{ + Prefix: garageGlobalBucketBootstrapHostsDirPath, + Recursive: true, + }, + ) + ) + + for objInfo := range objInfoCh { + + ctx := mctx.Annotate(ctx, "objectKey", objInfo.Key) + + if objInfo.Err != nil { + return nil, fmt.Errorf("listing objects: %w", objInfo.Err) + } + + obj, err := client.GetObject( + ctx, garage.GlobalBucket, objInfo.Key, minio.GetObjectOptions{}, + ) + + if err != nil { + return nil, fmt.Errorf("retrieving object %q: %w", objInfo.Key, err) + } + + var authedHost bootstrap.AuthenticatedHost + + err = json.NewDecoder(obj).Decode(&authedHost) + obj.Close() + + if err != nil { + d.logger.Warn(ctx, "object contains invalid json", err) + continue + } + + host, err := authedHost.Unwrap(b.CAPublicCredentials) + if err != nil { + d.logger.Warn(ctx, "host could not be authenticated", err) + } + + hosts[host.Name] = host + } + + return hosts, nil +}