From f9d033b89ff836c43699a68db1c1114994edb8ec Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Sun, 7 Jul 2024 20:01:10 +0200 Subject: [PATCH] Implement Daemon.CreateNetwork, but it's not yet used or tested --- docs/user/getting-started.md | 2 - go/cmd/entrypoint/admin.go | 2 +- go/cmd/entrypoint/garage_util.go | 2 +- go/daemon/child_garage.go | 10 +- go/daemon/daemon.go | 264 +++++++++++++++--- go/daemon/errors.go | 6 + go/daemon/global_bucket.go | 39 +++ go/daemon/jigs.go | 10 + go/daemon/jsonrpc2/errors.go | 6 + .../utils/with-1-data-1-empty-node-network.sh | 3 - 10 files changed, 299 insertions(+), 45 deletions(-) diff --git a/docs/user/getting-started.md b/docs/user/getting-started.md index 52aad6b..2b1212f 100644 --- a/docs/user/getting-started.md +++ b/docs/user/getting-started.md @@ -124,8 +124,6 @@ isle network join --bootstrap-path /path/to/bootstrap.json After a few moments you will have successfully joined the network! -TODO block the `network join` call until joining has succeeded, or display a failure reason. - [creating-a-new-network]: ../admin/creating-a-new-network.md [latest]: https://code.betamike.com/micropelago/isle/releases/latest diff --git a/go/cmd/entrypoint/admin.go b/go/cmd/entrypoint/admin.go index cc56e73..b2ddc44 100644 --- a/go/cmd/entrypoint/admin.go +++ b/go/cmd/entrypoint/admin.go @@ -190,7 +190,7 @@ var subCmdAdminCreateNetwork = subCmd{ logger.Info(ctx, "Applying garage layout") if err := daemon.GarageApplyLayout( - ctx, logger, hostBootstrap, daemonConfig, + ctx, logger, daemonConfig, hostBootstrap, ); err != nil { return fmt.Errorf("applying garage layout: %w", err) } diff --git a/go/cmd/entrypoint/garage_util.go b/go/cmd/entrypoint/garage_util.go index 826960d..737f954 100644 --- a/go/cmd/entrypoint/garage_util.go +++ b/go/cmd/entrypoint/garage_util.go @@ -19,7 +19,7 @@ func garageInitializeGlobalBucket( garage.S3APICredentials, error, ) { adminClient := daemon.NewGarageAdminClient( - logger, hostBootstrap, daemonConfig, + logger, daemonConfig, hostBootstrap, ) creds, err := adminClient.CreateS3APICredentials( diff --git a/go/daemon/child_garage.go b/go/daemon/child_garage.go index 5abc2a3..5ce4f70 100644 --- a/go/daemon/child_garage.go +++ b/go/daemon/child_garage.go @@ -23,8 +23,8 @@ func garageAdminClientLogger(logger *mlog.Logger) *mlog.Logger { // or it will _panic_ if there is no local instance configured. func NewGarageAdminClient( logger *mlog.Logger, + daemonConfig Config, hostBootstrap bootstrap.Bootstrap, - config Config, ) *garage.AdminClient { thisHost := hostBootstrap.ThisHost() @@ -33,7 +33,7 @@ func NewGarageAdminClient( garageAdminClientLogger(logger), net.JoinHostPort( thisHost.IP().String(), - strconv.Itoa(config.Storage.Allocations[0].AdminPort), + strconv.Itoa(daemonConfig.Storage.Allocations[0].AdminPort), ), hostBootstrap.Garage.AdminToken, ) @@ -181,15 +181,15 @@ func garagePmuxProcConfigs( func GarageApplyLayout( ctx context.Context, logger *mlog.Logger, + daemonConfig Config, hostBootstrap bootstrap.Bootstrap, - config Config, ) error { var ( - adminClient = NewGarageAdminClient(logger, hostBootstrap, config) + adminClient = NewGarageAdminClient(logger, daemonConfig, hostBootstrap) thisHost = hostBootstrap.ThisHost() hostName = thisHost.Name - allocs = config.Storage.Allocations + allocs = daemonConfig.Storage.Allocations peers = make([]garage.PeerLayout, len(allocs)) ) diff --git a/go/daemon/daemon.go b/go/daemon/daemon.go index 53c07f6..3ca6342 100644 --- a/go/daemon/daemon.go +++ b/go/daemon/daemon.go @@ -9,18 +9,51 @@ import ( "fmt" "io" "io/fs" + "isle/admin" "isle/bootstrap" + "isle/daemon/jsonrpc2" + "isle/garage" + "isle/nebula" + "net" "os" + "regexp" "sync" "time" "dev.mediocregopher.com/mediocre-go-lib.git/mlog" ) +var hostNameRegexp = regexp.MustCompile(`^[a-z][a-z0-9\-]*$`) + +func validateHostName(name string) error { + + if !hostNameRegexp.MatchString(name) { + return errors.New("a host's name must start with a letter and only contain letters, numbers, and dashes") + } + + return nil +} + // Daemon presents all functionality required for client frontends to interact // with isle, typically via the unix socket. type Daemon interface { + // CreateNetwork will initialize a new network using the given parameters. + // - name: Human-readable name of the network. + // - domain: Primary domain name that network services are served under. + // - ipNetStr: An IP + subnet mask which represents both the IP of this + // first host in the network, as well as the overall range of possible IPs + // in the network. + // - hostName: The name of this first host in the network. + // + // An Admin instance is returned, which is necessary to perform admin + // actions in the future. + CreateNetwork( + ctx context.Context, name, domain, ipNetStr, hostName string, + ) ( + admin.Admin, error, + ) + // JoinNetwork joins the Daemon to an existing network using the given // Bootstrap. // @@ -141,14 +174,21 @@ func NewDaemon( return nil, fmt.Errorf( "loading bootstrap from %q: %w", bootstrapFilePath, err, ) - } else if err := d.initialize(currBootstrap); err != nil { + } else if err := d.initialize(currBootstrap, nil); err != nil { return nil, fmt.Errorf("initializing with bootstrap: %w", err) } return d, nil } -func (d *daemon) initialize(currBootstrap bootstrap.Bootstrap) error { +// initialize must be called with d.l write lock held, _but_ the lock should be +// released just after initialize returns. +// +// readyCh will be written to everytime daemon changes state to daemonStateOk, +// unless it is nil or blocks. +func (d *daemon) initialize( + currBootstrap bootstrap.Bootstrap, readyCh chan<- struct{}, +) error { // we update this Host's data using whatever configuration has been provided // by the daemon config. This way the daemon has the most up-to-date // possible bootstrap. This updated bootstrap will later get updated in @@ -179,7 +219,7 @@ func (d *daemon) initialize(currBootstrap bootstrap.Bootstrap) error { d.wg.Add(1) go func() { defer d.wg.Done() - d.restartLoop(ctx) + d.restartLoop(ctx, readyCh) d.logger.Debug(ctx, "Daemon restart loop stopped") }() @@ -287,7 +327,64 @@ func (d *daemon) watchForChanges(ctx context.Context) bootstrap.Bootstrap { } } -func (d *daemon) restartLoop(ctx context.Context) { +func (d *daemon) postInit(ctx context.Context) bool { + if len(d.daemonConfig.Storage.Allocations) > 0 { + if !until( + ctx, + d.logger, + "Applying garage layout", + func(ctx context.Context) error { + return GarageApplyLayout( + ctx, d.logger, d.daemonConfig, d.currBootstrap, + ) + }, + ) { + return false + } + } + + // This is only necessary during network creation, otherwise the bootstrap + // should already have these credentials built in. + // + // TODO this is pretty hacky, but there doesn't seem to be a better way to + // manage it at the moment. + if d.currBootstrap.Garage.GlobalBucketS3APICredentials == (garage.S3APICredentials{}) { + var garageGlobalBucketCreds garage.S3APICredentials + if !until( + ctx, + d.logger, + "Initializing garage shared global bucket", + func(ctx context.Context) error { + var err error + garageGlobalBucketCreds, err = garageInitializeGlobalBucket( + ctx, d.logger, d.daemonConfig, d.currBootstrap, + ) + return err + }, + ) { + return false + } + + d.l.Lock() + d.currBootstrap.Garage.GlobalBucketS3APICredentials = garageGlobalBucketCreds + d.l.Unlock() + } + + if !until( + ctx, + d.logger, + "Updating host info in garage", + func(ctx context.Context) error { + return d.putGarageBoostrapHost(ctx) + }, + ) { + return false + } + + return true +} + +func (d *daemon) restartLoop(ctx context.Context, readyCh chan<- struct{}) { wait := func(d time.Duration) bool { select { case <-ctx.Done(): @@ -297,15 +394,22 @@ func (d *daemon) restartLoop(ctx context.Context) { } } + ready := func() { + select { + case readyCh <- struct{}{}: + default: + } + } + for { if ctx.Err() != nil { return } - d.logger.Info(ctx, "Creating new daemon") + d.logger.Info(ctx, "Creating child processes") children, err := NewChildren( ctx, - d.logger.WithNamespace("daemon"), + d.logger.WithNamespace("children"), d.daemonConfig, d.currBootstrap, d.envBinDirPath, @@ -321,36 +425,21 @@ func (d *daemon) restartLoop(ctx context.Context) { continue } + d.logger.Info(ctx, "Child processes created") + d.l.Lock() d.children = children + d.l.Unlock() + + if !d.postInit(ctx) { + return + } + + d.l.Lock() d.state = daemonStateOk d.l.Unlock() - if len(d.daemonConfig.Storage.Allocations) > 0 { - if !until( - ctx, - d.logger, - "Applying garage layout", - func(ctx context.Context) error { - return GarageApplyLayout( - ctx, d.logger, d.currBootstrap, d.daemonConfig, - ) - }, - ) { - return - } - } - - if !until( - ctx, - d.logger, - "Updating host info in garage", - func(ctx context.Context) error { - return d.putGarageBoostrapHost(ctx) - }, - ) { - return - } + ready() newBootstrap := d.watchForChanges(ctx) if ctx.Err() != nil { @@ -367,20 +456,129 @@ func (d *daemon) restartLoop(ctx context.Context) { if err := d.children.Shutdown(); err != nil { d.logger.Fatal(ctx, "Failed to cleanly shutdown children, there may be orphaned child processes", err) } + + // in case context was canceled while shutting the Children down, we + // don't want the Shutdown method to re-attempt calling Shutdown on + // it. + d.children = nil } } +func (d *daemon) CreateNetwork( + ctx context.Context, + name, domain, ipNetStr, hostName string, +) ( + admin.Admin, error, +) { + ip, subnet, err := net.ParseCIDR(ipNetStr) + if err != nil { + return admin.Admin{}, jsonrpc2.NewInvalidParamsError( + "parsing %q as a CIDR: %v", ipNetStr, err, + ) + } + + if err := validateHostName(hostName); err != nil { + return admin.Admin{}, jsonrpc2.NewInvalidParamsError( + "invalid hostname: %v", err, + ) + } + + nebulaCACreds, err := nebula.NewCACredentials(domain, subnet) + if err != nil { + return admin.Admin{}, fmt.Errorf("creating nebula CA cert: %w", err) + } + + adm := admin.Admin{ + CreationParams: admin.CreationParams{ + ID: randStr(32), + Name: name, + Domain: domain, + }, + } + + garageBootstrap := bootstrap.Garage{ + RPCSecret: randStr(32), + AdminToken: randStr(32), + } + + hostBootstrap, err := bootstrap.New( + nebulaCACreds, + adm.CreationParams, + garageBootstrap, + hostName, + ip, + ) + if err != nil { + return adm, fmt.Errorf("initializing bootstrap data: %w", err) + } + + d.l.Lock() + + if d.state != daemonStateNoNetwork { + d.l.Unlock() + return adm, ErrAlreadyJoined + } + + if len(d.daemonConfig.Storage.Allocations) < 3 { + d.l.Unlock() + return adm, ErrInvalidConfig.WithData( + "At least three storage allocations are required.", + ) + } + + readyCh := make(chan struct{}, 1) + + err = d.initialize(hostBootstrap, readyCh) + d.l.Unlock() + if err != nil { + return adm, fmt.Errorf("initializing daemon: %w", err) + } + + select { + case <-readyCh: + case <-ctx.Done(): + return adm, ctx.Err() + } + + // As part of postInit, which is called prior to ready(), the restartLoop + // will check if the global bucket creds have been created yet or not, and + // create them if so. So once ready() is called we can get the created creds + // from the currBootstrap + d.l.RLock() + garageGlobalBucketCreds := d.currBootstrap.Garage.GlobalBucketS3APICredentials + d.l.RUnlock() + + adm.Nebula.CACredentials = nebulaCACreds + adm.Garage.RPCSecret = hostBootstrap.Garage.RPCSecret + adm.Garage.GlobalBucketS3APICredentials = garageGlobalBucketCreds + + return adm, nil +} + func (d *daemon) JoinNetwork( ctx context.Context, newBootstrap bootstrap.Bootstrap, ) error { d.l.Lock() - defer d.l.Unlock() if d.state != daemonStateNoNetwork { + d.l.Unlock() return ErrAlreadyJoined } - return d.initialize(newBootstrap) + readyCh := make(chan struct{}, 1) + + err := d.initialize(newBootstrap, readyCh) + d.l.Unlock() + if err != nil { + return fmt.Errorf("initializing daemon: %w", err) + } + + select { + case <-readyCh: + return nil + case <-ctx.Done(): + return ctx.Err() + } } func (d *daemon) GetGarageBootstrapHosts( diff --git a/go/daemon/errors.go b/go/daemon/errors.go index c9e1aa6..2378e67 100644 --- a/go/daemon/errors.go +++ b/go/daemon/errors.go @@ -18,4 +18,10 @@ var ( // ErrAlreadyJoined is returned when the daemon is instructed to create or // join a new network, but it is already joined to a network. ErrAlreadyJoined = jsonrpc2.NewError(4, "Already joined to a network") + + // ErrInvalidConfig is returned when the daemon's configuration is invalid + // for an operation being attempted. + // + // The Data field will be a string containing further details. + ErrInvalidConfig = jsonrpc2.NewError(5, "Invalid daemon config") ) diff --git a/go/daemon/global_bucket.go b/go/daemon/global_bucket.go index 993b804..6928d30 100644 --- a/go/daemon/global_bucket.go +++ b/go/daemon/global_bucket.go @@ -20,6 +20,45 @@ const ( garageGlobalBucketBootstrapHostsDirPath = "bootstrap/hosts" ) +func garageInitializeGlobalBucket( + ctx context.Context, + logger *mlog.Logger, + daemonConfig Config, + hostBootstrap bootstrap.Bootstrap, +) ( + garage.S3APICredentials, error, +) { + adminClient := NewGarageAdminClient( + logger, daemonConfig, hostBootstrap, + ) + + creds, err := adminClient.CreateS3APICredentials( + ctx, garage.GlobalBucketS3APICredentialsName, + ) + if err != nil { + return creds, fmt.Errorf("creating global bucket credentials: %w", err) + } + + bucketID, err := adminClient.CreateBucket(ctx, garage.GlobalBucket) + if err != nil { + return creds, fmt.Errorf("creating global bucket: %w", err) + } + + if err := adminClient.GrantBucketPermissions( + ctx, + bucketID, + creds.ID, + garage.BucketPermissionRead, + garage.BucketPermissionWrite, + ); err != nil { + return creds, fmt.Errorf( + "granting permissions to shared global bucket key: %w", err, + ) + } + + return creds, nil +} + // putGarageBoostrapHost places the .json.signed file for this host // into garage so that other hosts are able to see relevant configuration for // it. diff --git a/go/daemon/jigs.go b/go/daemon/jigs.go index 3e46ce2..f02fc99 100644 --- a/go/daemon/jigs.go +++ b/go/daemon/jigs.go @@ -2,6 +2,8 @@ package daemon import ( "context" + "crypto/rand" + "encoding/hex" "time" "dev.mediocregopher.com/mediocre-go-lib.git/mlog" @@ -28,3 +30,11 @@ func until( time.Sleep(1 * time.Second) } } + +func randStr(l int) string { + b := make([]byte, l) + if _, err := rand.Read(b); err != nil { + panic(err) + } + return hex.EncodeToString(b) +} diff --git a/go/daemon/jsonrpc2/errors.go b/go/daemon/jsonrpc2/errors.go index 14c0374..947ffac 100644 --- a/go/daemon/jsonrpc2/errors.go +++ b/go/daemon/jsonrpc2/errors.go @@ -73,3 +73,9 @@ func (e Error) Is(target error) bool { } return false } + +// WithData returns a copy of the Error with the Data file overwritten. +func (e Error) WithData(data any) Error { + e.Data = data + return e +} diff --git a/tests/utils/with-1-data-1-empty-node-network.sh b/tests/utils/with-1-data-1-empty-node-network.sh index 5870fdc..20ecfad 100644 --- a/tests/utils/with-1-data-1-empty-node-network.sh +++ b/tests/utils/with-1-data-1-empty-node-network.sh @@ -90,8 +90,5 @@ EOF echo "Joining secondus to the network" isle network join -b "$secondus_bootstrap" - - echo "Waiting for secondus daemon to join" - while ! isle hosts list >/dev/null; do sleep 1; done ) fi