micropelago/isle
4
1
Fork 0
Code Issues 1 Pull Requests Projects Releases 5 Activity

Move some Bootstrap methods onto Daemon

Browse Source
This commit is contained in:
Brian Picciano 2024-06-17 22:15:28 +02:00
parent a8893e4fc6
commit 4e5d3b28ab
7 changed files with 155 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
go/bootstrap/garage_global_bucket.go
View File

@ -1,12 +1,10 @@
package bootstrap
import (
"bytes"
"context"
"encoding/json"
"fmt"
"isle/garage"
"isle/nebula"
"path/filepath"
"github.com/mediocregopher/mediocre-go-lib/v2/mctx"
@ -14,56 +12,13 @@ import (
"github.com/minio/minio-go/v7"
)
// Paths within garage's global bucket
// Paths within garage's global bucket.
//
// TODO this is getting moved into daemon package.
const (
garageGlobalBucketBootstrapHostsDirPath = "bootstrap/hosts"
)
// PutGarageBoostrapHost places the <hostname>.json.signed file for this host
// into garage so that other hosts are able to see relevant configuration for
// it.
func (b Bootstrap) PutGarageBoostrapHost(ctx context.Context) error {
var (
host = b.ThisHost()
client = b.GlobalBucketS3APIClient()
)
configured, err := nebula.Sign(
host.HostConfigured, b.PrivateCredentials.SigningPrivateKey,
)
if err != nil {
return fmt.Errorf("signing host configured data: %w", err)
}
hostB, err := json.Marshal(AuthenticatedHost{
Assigned: b.SignedHostAssigned,
Configured: configured,
})
if err != nil {
return fmt.Errorf("encoding host data: %w", err)
}
filePath := filepath.Join(
garageGlobalBucketBootstrapHostsDirPath,
host.Name+".json.signed",
)
_, err = client.PutObject(
ctx,
garage.GlobalBucket,
filePath,
bytes.NewReader(hostB),
int64(len(hostB)),
minio.PutObjectOptions{},
)
if err != nil {
return fmt.Errorf("writing to %q in global bucket: %w", filePath, err)
}
return nil
}
// RemoveGarageBootstrapHost removes the <hostname>.json.signed for the given
// host from garage.
//
@ -85,6 +40,8 @@ func RemoveGarageBootstrapHost(
// GetGarageBootstrapHosts loads the <hostname>.json.signed file for all hosts
// stored in garage.
//
// Deprecated: should use the method off Daemon instead.
func (b Bootstrap) GetGarageBootstrapHosts(
ctx context.Context,
logger *mlog.Logger,

3
go/cmd/entrypoint/admin.go
View File

@ -173,6 +173,7 @@ var subCmdAdminCreateNetwork = subCmd{
hostBootstrap,
envRuntimeDirPath,
envBinDirPath,
envStateDirPath,
&daemon.Opts{
// SkipHostBootstrapPush is required, because the global bucket
// hasn't actually been initialized yet, so there's nowhere to
@ -208,7 +209,7 @@ var subCmdAdminCreateNetwork = subCmd{
// rewrite the bootstrap now that the global bucket creds have been
// added to it.
if err := writeBootstrapToDataDir(hostBootstrap); err != nil {
if err := writeBootstrapToStateDir(hostBootstrap); err != nil {
return fmt.Errorf("writing bootstrap file: %w", err)
}

2
go/cmd/entrypoint/bootstrap_util.go
View File

@ -27,7 +27,7 @@ func loadHostBootstrap() (bootstrap.Bootstrap, error) {
return hostBootstrap, nil
}
func writeBootstrapToDataDir(hostBootstrap bootstrap.Bootstrap) error {
func writeBootstrapToStateDir(hostBootstrap bootstrap.Bootstrap) error {
path := bootstrap.StateDirPath(envStateDirPath)
dirPath := filepath.Dir(path)

12
go/cmd/entrypoint/daemon.go
View File

@ -41,6 +41,7 @@ import (
func reloadBootstrap(
ctx context.Context,
logger *mlog.Logger,
daemonInst daemon.Daemon,
hostBootstrap bootstrap.Bootstrap,
) (
bootstrap.Bootstrap, bool, error,
@ -48,7 +49,7 @@ func reloadBootstrap(
thisHost := hostBootstrap.ThisHost()
newHosts, err := hostBootstrap.GetGarageBootstrapHosts(ctx, logger)
newHosts, err := daemonInst.GetGarageBootstrapHosts(ctx)
if err != nil {
return bootstrap.Bootstrap{}, false, fmt.Errorf("getting hosts from garage: %w", err)
}
@ -73,7 +74,7 @@ func reloadBootstrap(
hostBootstrap.Hosts = newHosts
if err := writeBootstrapToDataDir(hostBootstrap); err != nil {
if err := writeBootstrapToStateDir(hostBootstrap); err != nil {
return bootstrap.Bootstrap{}, false, fmt.Errorf("writing new bootstrap to data dir: %w", err)
}
@ -99,6 +100,7 @@ func runDaemonPmuxOnce(
hostBootstrap,
envRuntimeDirPath,
envBinDirPath,
envStateDirPath,
nil,
)
if err != nil {
@ -131,7 +133,9 @@ func runDaemonPmuxOnce(
err error
)
if hostBootstrap, changed, err = reloadBootstrap(ctx, logger, hostBootstrap); err != nil {
if hostBootstrap, changed, err = reloadBootstrap(
ctx, logger, daemonInst, hostBootstrap,
); err != nil {
return bootstrap.Bootstrap{}, fmt.Errorf("reloading bootstrap: %w", err)
} else if changed {
@ -236,7 +240,7 @@ var subCmdDaemon = subCmd{
// If the bootstrap file is not being stored in the data dir, copy
// it there, so it can be loaded from there next time.
if err := writeBootstrapToDataDir(hostBootstrap); err != nil {
if err := writeBootstrapToStateDir(hostBootstrap); err != nil {
return fmt.Errorf("writing bootstrap.json to data dir: %w", err)
}
}

10
go/cmd/entrypoint/daemon_util.go
View File

@ -8,12 +8,10 @@ import (
)
func coalesceDaemonConfigAndBootstrap(
hostBootstrap bootstrap.Bootstrap,
daemonConfig daemon.Config,
hostBootstrap bootstrap.Bootstrap, daemonConfig daemon.Config,
) (
bootstrap.Bootstrap, error,
) {
host := bootstrap.Host{
HostAssigned: hostBootstrap.HostAssigned,
HostConfigured: bootstrap.HostConfigured{
@ -29,7 +27,9 @@ func coalesceDaemonConfigAndBootstrap(
id, rpcPort, err := garagesrv.InitAlloc(alloc.MetaPath, alloc.RPCPort)
if err != nil {
return bootstrap.Bootstrap{}, fmt.Errorf("initializing alloc at %q: %w", alloc.MetaPath, err)
return bootstrap.Bootstrap{}, fmt.Errorf(
"initializing alloc at %q: %w", alloc.MetaPath, err,
)
}
host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
@ -44,7 +44,7 @@ func coalesceDaemonConfigAndBootstrap(
hostBootstrap.Hosts[host.Name] = host
if err := writeBootstrapToDataDir(hostBootstrap); err != nil {
if err := writeBootstrapToStateDir(hostBootstrap); err != nil {
return bootstrap.Bootstrap{}, fmt.Errorf("writing bootstrap file: %w", err)
}

14
go/daemon/daemon.go
View File

@ -18,6 +18,7 @@ type daemon struct {
logger *mlog.Logger
config Config
hostBootstrap bootstrap.Bootstrap
stateDirPath string
opts Opts
pmuxCancelFn context.CancelFunc
@ -28,6 +29,14 @@ type daemon struct {
// with isle, typically via the unix socket.
type Daemon interface {
// GetGarageBootstrapHosts loads (and verifies) the <hostname>.json.signed
// file for all hosts stored in garage.
GetGarageBootstrapHosts(
ctx context.Context,
) (
map[string]bootstrap.Host, error,
)
// Shutdown blocks until all resources held or created by the daemon,
// including child processes it has started, have been cleaned up, or until
// the context is canceled.
@ -72,7 +81,7 @@ func New(
logger *mlog.Logger,
config Config,
hostBootstrap bootstrap.Bootstrap,
runtimeDirPath, binDirPath string,
runtimeDirPath, binDirPath, stateDirPath string,
opts *Opts,
) (
Daemon, error,
@ -116,6 +125,7 @@ func New(
logger: logger,
config: config,
hostBootstrap: hostBootstrap,
stateDirPath: stateDirPath,
opts: *opts,
pmuxCancelFn: pmuxCancelFn,
pmuxStoppedCh: make(chan struct{}),
@ -175,7 +185,7 @@ func (d *daemon) postPmuxInit(ctx context.Context) error {
if !d.opts.SkipHostBootstrapPush {
if err := until(ctx, func(ctx context.Context) error {
if err := d.hostBootstrap.PutGarageBoostrapHost(ctx); err != nil {
if err := d.putGarageBoostrapHost(ctx); err != nil {
d.logger.Error(ctx, "updating host info in garage", err)
return err
}

122
go/daemon/global_bucket.go Normal file
View File

@ -0,0 +1,122 @@
package daemon
import (
"bytes"
"context"
"encoding/json"
"fmt"
"isle/bootstrap"
"isle/garage"
"isle/nebula"
"path/filepath"
"github.com/mediocregopher/mediocre-go-lib/v2/mctx"
"github.com/minio/minio-go/v7"
)
// Paths within garage's global bucket.
const (
garageGlobalBucketBootstrapHostsDirPath = "bootstrap/hosts"
)
// putGarageBoostrapHost places the <hostname>.json.signed file for this host
// into garage so that other hosts are able to see relevant configuration for
// it.
func (d *daemon) putGarageBoostrapHost(ctx context.Context) error {
var (
b = d.hostBootstrap
host = b.ThisHost()
client = b.GlobalBucketS3APIClient()
)
configured, err := nebula.Sign(
host.HostConfigured, b.PrivateCredentials.SigningPrivateKey,
)
if err != nil {
return fmt.Errorf("signing host configured data: %w", err)
}
hostB, err := json.Marshal(bootstrap.AuthenticatedHost{
Assigned: b.SignedHostAssigned,
Configured: configured,
})
if err != nil {
return fmt.Errorf("encoding host data: %w", err)
}
filePath := filepath.Join(
garageGlobalBucketBootstrapHostsDirPath,
host.Name+".json.signed",
)
_, err = client.PutObject(
ctx,
garage.GlobalBucket,
filePath,
bytes.NewReader(hostB),
int64(len(hostB)),
minio.PutObjectOptions{},
)
if err != nil {
return fmt.Errorf("writing to %q in global bucket: %w", filePath, err)
}
return nil
}
func (d *daemon) GetGarageBootstrapHosts(
ctx context.Context,
) (
map[string]bootstrap.Host, error,
) {
var (
b = d.hostBootstrap
client = b.GlobalBucketS3APIClient()
hosts = map[string]bootstrap.Host{}
objInfoCh = client.ListObjects(
ctx, garage.GlobalBucket,
minio.ListObjectsOptions{
Prefix: garageGlobalBucketBootstrapHostsDirPath,
Recursive: true,
},
)
)
for objInfo := range objInfoCh {
ctx := mctx.Annotate(ctx, "objectKey", objInfo.Key)
if objInfo.Err != nil {
return nil, fmt.Errorf("listing objects: %w", objInfo.Err)
}
obj, err := client.GetObject(
ctx, garage.GlobalBucket, objInfo.Key, minio.GetObjectOptions{},
)
if err != nil {
return nil, fmt.Errorf("retrieving object %q: %w", objInfo.Key, err)
}
var authedHost bootstrap.AuthenticatedHost
err = json.NewDecoder(obj).Decode(&authedHost)
obj.Close()
if err != nil {
d.logger.Warn(ctx, "object contains invalid json", err)
continue
}
host, err := authedHost.Unwrap(b.CAPublicCredentials)
if err != nil {
d.logger.Warn(ctx, "host could not be authenticated", err)
}
hosts[host.Name] = host
}
return hosts, nil
}