Implement Daemon.CreateNetwork, but it's not yet used or tested

This commit is contained in:
Brian Picciano 2024-07-07 20:01:10 +02:00
parent ce5df164e1
commit f9d033b89f
10 changed files with 299 additions and 45 deletions

View File

@ -124,8 +124,6 @@ isle network join --bootstrap-path /path/to/bootstrap.json
After a few moments you will have successfully joined the network! After a few moments you will have successfully joined the network!
TODO block the `network join` call until joining has succeeded, or display a failure reason.
[creating-a-new-network]: ../admin/creating-a-new-network.md [creating-a-new-network]: ../admin/creating-a-new-network.md
[latest]: https://code.betamike.com/micropelago/isle/releases/latest [latest]: https://code.betamike.com/micropelago/isle/releases/latest

View File

@ -190,7 +190,7 @@ var subCmdAdminCreateNetwork = subCmd{
logger.Info(ctx, "Applying garage layout") logger.Info(ctx, "Applying garage layout")
if err := daemon.GarageApplyLayout( if err := daemon.GarageApplyLayout(
ctx, logger, hostBootstrap, daemonConfig, ctx, logger, daemonConfig, hostBootstrap,
); err != nil { ); err != nil {
return fmt.Errorf("applying garage layout: %w", err) return fmt.Errorf("applying garage layout: %w", err)
} }

View File

@ -19,7 +19,7 @@ func garageInitializeGlobalBucket(
garage.S3APICredentials, error, garage.S3APICredentials, error,
) { ) {
adminClient := daemon.NewGarageAdminClient( adminClient := daemon.NewGarageAdminClient(
logger, hostBootstrap, daemonConfig, logger, daemonConfig, hostBootstrap,
) )
creds, err := adminClient.CreateS3APICredentials( creds, err := adminClient.CreateS3APICredentials(

View File

@ -23,8 +23,8 @@ func garageAdminClientLogger(logger *mlog.Logger) *mlog.Logger {
// or it will _panic_ if there is no local instance configured. // or it will _panic_ if there is no local instance configured.
func NewGarageAdminClient( func NewGarageAdminClient(
logger *mlog.Logger, logger *mlog.Logger,
daemonConfig Config,
hostBootstrap bootstrap.Bootstrap, hostBootstrap bootstrap.Bootstrap,
config Config,
) *garage.AdminClient { ) *garage.AdminClient {
thisHost := hostBootstrap.ThisHost() thisHost := hostBootstrap.ThisHost()
@ -33,7 +33,7 @@ func NewGarageAdminClient(
garageAdminClientLogger(logger), garageAdminClientLogger(logger),
net.JoinHostPort( net.JoinHostPort(
thisHost.IP().String(), thisHost.IP().String(),
strconv.Itoa(config.Storage.Allocations[0].AdminPort), strconv.Itoa(daemonConfig.Storage.Allocations[0].AdminPort),
), ),
hostBootstrap.Garage.AdminToken, hostBootstrap.Garage.AdminToken,
) )
@ -181,15 +181,15 @@ func garagePmuxProcConfigs(
func GarageApplyLayout( func GarageApplyLayout(
ctx context.Context, ctx context.Context,
logger *mlog.Logger, logger *mlog.Logger,
daemonConfig Config,
hostBootstrap bootstrap.Bootstrap, hostBootstrap bootstrap.Bootstrap,
config Config,
) error { ) error {
var ( var (
adminClient = NewGarageAdminClient(logger, hostBootstrap, config) adminClient = NewGarageAdminClient(logger, daemonConfig, hostBootstrap)
thisHost = hostBootstrap.ThisHost() thisHost = hostBootstrap.ThisHost()
hostName = thisHost.Name hostName = thisHost.Name
allocs = config.Storage.Allocations allocs = daemonConfig.Storage.Allocations
peers = make([]garage.PeerLayout, len(allocs)) peers = make([]garage.PeerLayout, len(allocs))
) )

View File

@ -9,18 +9,51 @@ import (
"fmt" "fmt"
"io" "io"
"io/fs" "io/fs"
"isle/admin"
"isle/bootstrap" "isle/bootstrap"
"isle/daemon/jsonrpc2"
"isle/garage"
"isle/nebula"
"net"
"os" "os"
"regexp"
"sync" "sync"
"time" "time"
"dev.mediocregopher.com/mediocre-go-lib.git/mlog" "dev.mediocregopher.com/mediocre-go-lib.git/mlog"
) )
var hostNameRegexp = regexp.MustCompile(`^[a-z][a-z0-9\-]*$`)
func validateHostName(name string) error {
if !hostNameRegexp.MatchString(name) {
return errors.New("a host's name must start with a letter and only contain letters, numbers, and dashes")
}
return nil
}
// Daemon presents all functionality required for client frontends to interact // Daemon presents all functionality required for client frontends to interact
// with isle, typically via the unix socket. // with isle, typically via the unix socket.
type Daemon interface { type Daemon interface {
// CreateNetwork will initialize a new network using the given parameters.
// - name: Human-readable name of the network.
// - domain: Primary domain name that network services are served under.
// - ipNetStr: An IP + subnet mask which represents both the IP of this
// first host in the network, as well as the overall range of possible IPs
// in the network.
// - hostName: The name of this first host in the network.
//
// An Admin instance is returned, which is necessary to perform admin
// actions in the future.
CreateNetwork(
ctx context.Context, name, domain, ipNetStr, hostName string,
) (
admin.Admin, error,
)
// JoinNetwork joins the Daemon to an existing network using the given // JoinNetwork joins the Daemon to an existing network using the given
// Bootstrap. // Bootstrap.
// //
@ -141,14 +174,21 @@ func NewDaemon(
return nil, fmt.Errorf( return nil, fmt.Errorf(
"loading bootstrap from %q: %w", bootstrapFilePath, err, "loading bootstrap from %q: %w", bootstrapFilePath, err,
) )
} else if err := d.initialize(currBootstrap); err != nil { } else if err := d.initialize(currBootstrap, nil); err != nil {
return nil, fmt.Errorf("initializing with bootstrap: %w", err) return nil, fmt.Errorf("initializing with bootstrap: %w", err)
} }
return d, nil return d, nil
} }
func (d *daemon) initialize(currBootstrap bootstrap.Bootstrap) error { // initialize must be called with d.l write lock held, _but_ the lock should be
// released just after initialize returns.
//
// readyCh will be written to everytime daemon changes state to daemonStateOk,
// unless it is nil or blocks.
func (d *daemon) initialize(
currBootstrap bootstrap.Bootstrap, readyCh chan<- struct{},
) error {
// we update this Host's data using whatever configuration has been provided // we update this Host's data using whatever configuration has been provided
// by the daemon config. This way the daemon has the most up-to-date // by the daemon config. This way the daemon has the most up-to-date
// possible bootstrap. This updated bootstrap will later get updated in // possible bootstrap. This updated bootstrap will later get updated in
@ -179,7 +219,7 @@ func (d *daemon) initialize(currBootstrap bootstrap.Bootstrap) error {
d.wg.Add(1) d.wg.Add(1)
go func() { go func() {
defer d.wg.Done() defer d.wg.Done()
d.restartLoop(ctx) d.restartLoop(ctx, readyCh)
d.logger.Debug(ctx, "Daemon restart loop stopped") d.logger.Debug(ctx, "Daemon restart loop stopped")
}() }()
@ -287,7 +327,64 @@ func (d *daemon) watchForChanges(ctx context.Context) bootstrap.Bootstrap {
} }
} }
func (d *daemon) restartLoop(ctx context.Context) { func (d *daemon) postInit(ctx context.Context) bool {
if len(d.daemonConfig.Storage.Allocations) > 0 {
if !until(
ctx,
d.logger,
"Applying garage layout",
func(ctx context.Context) error {
return GarageApplyLayout(
ctx, d.logger, d.daemonConfig, d.currBootstrap,
)
},
) {
return false
}
}
// This is only necessary during network creation, otherwise the bootstrap
// should already have these credentials built in.
//
// TODO this is pretty hacky, but there doesn't seem to be a better way to
// manage it at the moment.
if d.currBootstrap.Garage.GlobalBucketS3APICredentials == (garage.S3APICredentials{}) {
var garageGlobalBucketCreds garage.S3APICredentials
if !until(
ctx,
d.logger,
"Initializing garage shared global bucket",
func(ctx context.Context) error {
var err error
garageGlobalBucketCreds, err = garageInitializeGlobalBucket(
ctx, d.logger, d.daemonConfig, d.currBootstrap,
)
return err
},
) {
return false
}
d.l.Lock()
d.currBootstrap.Garage.GlobalBucketS3APICredentials = garageGlobalBucketCreds
d.l.Unlock()
}
if !until(
ctx,
d.logger,
"Updating host info in garage",
func(ctx context.Context) error {
return d.putGarageBoostrapHost(ctx)
},
) {
return false
}
return true
}
func (d *daemon) restartLoop(ctx context.Context, readyCh chan<- struct{}) {
wait := func(d time.Duration) bool { wait := func(d time.Duration) bool {
select { select {
case <-ctx.Done(): case <-ctx.Done():
@ -297,15 +394,22 @@ func (d *daemon) restartLoop(ctx context.Context) {
} }
} }
ready := func() {
select {
case readyCh <- struct{}{}:
default:
}
}
for { for {
if ctx.Err() != nil { if ctx.Err() != nil {
return return
} }
d.logger.Info(ctx, "Creating new daemon") d.logger.Info(ctx, "Creating child processes")
children, err := NewChildren( children, err := NewChildren(
ctx, ctx,
d.logger.WithNamespace("daemon"), d.logger.WithNamespace("children"),
d.daemonConfig, d.daemonConfig,
d.currBootstrap, d.currBootstrap,
d.envBinDirPath, d.envBinDirPath,
@ -321,36 +425,21 @@ func (d *daemon) restartLoop(ctx context.Context) {
continue continue
} }
d.logger.Info(ctx, "Child processes created")
d.l.Lock() d.l.Lock()
d.children = children d.children = children
d.l.Unlock()
if !d.postInit(ctx) {
return
}
d.l.Lock()
d.state = daemonStateOk d.state = daemonStateOk
d.l.Unlock() d.l.Unlock()
if len(d.daemonConfig.Storage.Allocations) > 0 { ready()
if !until(
ctx,
d.logger,
"Applying garage layout",
func(ctx context.Context) error {
return GarageApplyLayout(
ctx, d.logger, d.currBootstrap, d.daemonConfig,
)
},
) {
return
}
}
if !until(
ctx,
d.logger,
"Updating host info in garage",
func(ctx context.Context) error {
return d.putGarageBoostrapHost(ctx)
},
) {
return
}
newBootstrap := d.watchForChanges(ctx) newBootstrap := d.watchForChanges(ctx)
if ctx.Err() != nil { if ctx.Err() != nil {
@ -367,20 +456,129 @@ func (d *daemon) restartLoop(ctx context.Context) {
if err := d.children.Shutdown(); err != nil { if err := d.children.Shutdown(); err != nil {
d.logger.Fatal(ctx, "Failed to cleanly shutdown children, there may be orphaned child processes", err) d.logger.Fatal(ctx, "Failed to cleanly shutdown children, there may be orphaned child processes", err)
} }
// in case context was canceled while shutting the Children down, we
// don't want the Shutdown method to re-attempt calling Shutdown on
// it.
d.children = nil
} }
} }
func (d *daemon) CreateNetwork(
ctx context.Context,
name, domain, ipNetStr, hostName string,
) (
admin.Admin, error,
) {
ip, subnet, err := net.ParseCIDR(ipNetStr)
if err != nil {
return admin.Admin{}, jsonrpc2.NewInvalidParamsError(
"parsing %q as a CIDR: %v", ipNetStr, err,
)
}
if err := validateHostName(hostName); err != nil {
return admin.Admin{}, jsonrpc2.NewInvalidParamsError(
"invalid hostname: %v", err,
)
}
nebulaCACreds, err := nebula.NewCACredentials(domain, subnet)
if err != nil {
return admin.Admin{}, fmt.Errorf("creating nebula CA cert: %w", err)
}
adm := admin.Admin{
CreationParams: admin.CreationParams{
ID: randStr(32),
Name: name,
Domain: domain,
},
}
garageBootstrap := bootstrap.Garage{
RPCSecret: randStr(32),
AdminToken: randStr(32),
}
hostBootstrap, err := bootstrap.New(
nebulaCACreds,
adm.CreationParams,
garageBootstrap,
hostName,
ip,
)
if err != nil {
return adm, fmt.Errorf("initializing bootstrap data: %w", err)
}
d.l.Lock()
if d.state != daemonStateNoNetwork {
d.l.Unlock()
return adm, ErrAlreadyJoined
}
if len(d.daemonConfig.Storage.Allocations) < 3 {
d.l.Unlock()
return adm, ErrInvalidConfig.WithData(
"At least three storage allocations are required.",
)
}
readyCh := make(chan struct{}, 1)
err = d.initialize(hostBootstrap, readyCh)
d.l.Unlock()
if err != nil {
return adm, fmt.Errorf("initializing daemon: %w", err)
}
select {
case <-readyCh:
case <-ctx.Done():
return adm, ctx.Err()
}
// As part of postInit, which is called prior to ready(), the restartLoop
// will check if the global bucket creds have been created yet or not, and
// create them if so. So once ready() is called we can get the created creds
// from the currBootstrap
d.l.RLock()
garageGlobalBucketCreds := d.currBootstrap.Garage.GlobalBucketS3APICredentials
d.l.RUnlock()
adm.Nebula.CACredentials = nebulaCACreds
adm.Garage.RPCSecret = hostBootstrap.Garage.RPCSecret
adm.Garage.GlobalBucketS3APICredentials = garageGlobalBucketCreds
return adm, nil
}
func (d *daemon) JoinNetwork( func (d *daemon) JoinNetwork(
ctx context.Context, newBootstrap bootstrap.Bootstrap, ctx context.Context, newBootstrap bootstrap.Bootstrap,
) error { ) error {
d.l.Lock() d.l.Lock()
defer d.l.Unlock()
if d.state != daemonStateNoNetwork { if d.state != daemonStateNoNetwork {
d.l.Unlock()
return ErrAlreadyJoined return ErrAlreadyJoined
} }
return d.initialize(newBootstrap) readyCh := make(chan struct{}, 1)
err := d.initialize(newBootstrap, readyCh)
d.l.Unlock()
if err != nil {
return fmt.Errorf("initializing daemon: %w", err)
}
select {
case <-readyCh:
return nil
case <-ctx.Done():
return ctx.Err()
}
} }
func (d *daemon) GetGarageBootstrapHosts( func (d *daemon) GetGarageBootstrapHosts(

View File

@ -18,4 +18,10 @@ var (
// ErrAlreadyJoined is returned when the daemon is instructed to create or // ErrAlreadyJoined is returned when the daemon is instructed to create or
// join a new network, but it is already joined to a network. // join a new network, but it is already joined to a network.
ErrAlreadyJoined = jsonrpc2.NewError(4, "Already joined to a network") ErrAlreadyJoined = jsonrpc2.NewError(4, "Already joined to a network")
// ErrInvalidConfig is returned when the daemon's configuration is invalid
// for an operation being attempted.
//
// The Data field will be a string containing further details.
ErrInvalidConfig = jsonrpc2.NewError(5, "Invalid daemon config")
) )

View File

@ -20,6 +20,45 @@ const (
garageGlobalBucketBootstrapHostsDirPath = "bootstrap/hosts" garageGlobalBucketBootstrapHostsDirPath = "bootstrap/hosts"
) )
func garageInitializeGlobalBucket(
ctx context.Context,
logger *mlog.Logger,
daemonConfig Config,
hostBootstrap bootstrap.Bootstrap,
) (
garage.S3APICredentials, error,
) {
adminClient := NewGarageAdminClient(
logger, daemonConfig, hostBootstrap,
)
creds, err := adminClient.CreateS3APICredentials(
ctx, garage.GlobalBucketS3APICredentialsName,
)
if err != nil {
return creds, fmt.Errorf("creating global bucket credentials: %w", err)
}
bucketID, err := adminClient.CreateBucket(ctx, garage.GlobalBucket)
if err != nil {
return creds, fmt.Errorf("creating global bucket: %w", err)
}
if err := adminClient.GrantBucketPermissions(
ctx,
bucketID,
creds.ID,
garage.BucketPermissionRead,
garage.BucketPermissionWrite,
); err != nil {
return creds, fmt.Errorf(
"granting permissions to shared global bucket key: %w", err,
)
}
return creds, nil
}
// putGarageBoostrapHost places the <hostname>.json.signed file for this host // putGarageBoostrapHost places the <hostname>.json.signed file for this host
// into garage so that other hosts are able to see relevant configuration for // into garage so that other hosts are able to see relevant configuration for
// it. // it.

View File

@ -2,6 +2,8 @@ package daemon
import ( import (
"context" "context"
"crypto/rand"
"encoding/hex"
"time" "time"
"dev.mediocregopher.com/mediocre-go-lib.git/mlog" "dev.mediocregopher.com/mediocre-go-lib.git/mlog"
@ -28,3 +30,11 @@ func until(
time.Sleep(1 * time.Second) time.Sleep(1 * time.Second)
} }
} }
func randStr(l int) string {
b := make([]byte, l)
if _, err := rand.Read(b); err != nil {
panic(err)
}
return hex.EncodeToString(b)
}

View File

@ -73,3 +73,9 @@ func (e Error) Is(target error) bool {
} }
return false return false
} }
// WithData returns a copy of the Error with the Data file overwritten.
func (e Error) WithData(data any) Error {
e.Data = data
return e
}

View File

@ -90,8 +90,5 @@ EOF
echo "Joining secondus to the network" echo "Joining secondus to the network"
isle network join -b "$secondus_bootstrap" isle network join -b "$secondus_bootstrap"
echo "Waiting for secondus daemon to join"
while ! isle hosts list >/dev/null; do sleep 1; done
) )
fi fi