From 93bdd3ebd4edaf5c87a45ee864e753afbbb51c68 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Sun, 16 Oct 2022 15:38:15 +0200 Subject: [PATCH] Update host's config in bootstrap using daemon.yml prior to starting pmux Previously if the `daemon.yml` of a host was changed it would first start up, load that new daemon.yml in, persist the new configuration for the host to garage using `update-garage-host`, pull that config back down and persist it to the bootstrap in `runDaemonPmuxOnce`, and restart all child processes so they get the new config. Now, once `daemon.yml` is loaded in we immediately merge it into and persist this host's bootstrap file, prior to ever starting child processes. This removes the necessity of restarting those process at start. This change also allows the bootstrap file to be the sole repository of information required to pick a garage node to connect to, since it is presumably always as up-to-date as it can possibly be. This allows for removing some more logic from `Env`. --- go-workspace/src/bootstrap/garage.go | 39 +++++++++++++ go-workspace/src/cmd/entrypoint/daemon.go | 57 ++++++++++++++++--- go-workspace/src/cmd/entrypoint/garage.go | 4 +- go-workspace/src/cmd/entrypoint/hosts.go | 8 +-- .../src/cmd/update-global-bucket/main.go | 38 +++---------- go-workspace/src/garage.go | 44 -------------- 6 files changed, 102 insertions(+), 88 deletions(-) delete mode 100644 go-workspace/src/garage.go diff --git a/go-workspace/src/bootstrap/garage.go b/go-workspace/src/bootstrap/garage.go index cdce6bd..6d33af4 100644 --- a/go-workspace/src/bootstrap/garage.go +++ b/go-workspace/src/bootstrap/garage.go @@ -2,6 +2,7 @@ package bootstrap import ( "cryptic-net/garage" + "fmt" ) // Paths within the bootstrap FS related to garage. @@ -45,3 +46,41 @@ func (b Bootstrap) GarageRPCPeerAddrs() []string { } return addrs } + +// ChooseGaragePeer returns a Peer for a garage instance from the network. It +// will prefer a garage instance on this particular host, if there is one, but +// will otherwise return a random endpoint. +func (b Bootstrap) ChooseGaragePeer() garage.Peer { + + thisHost := b.ThisHost() + + if thisHost.Garage != nil && len(thisHost.Garage.Instances) > 0 { + inst := thisHost.Garage.Instances[0] + return garage.Peer{ + IP: thisHost.Nebula.IP, + RPCPort: inst.RPCPort, + S3APIPort: inst.S3APIPort, + } + } + + for _, peer := range b.GaragePeers() { + return peer + } + + panic("no garage instances configured") +} + +// GlobalBucketS3APIClient returns an S3 client pre-configured with access to +// the global bucket. +func (b Bootstrap) GlobalBucketS3APIClient() (garage.S3APIClient, error) { + + addr := b.ChooseGaragePeer().S3APIAddr() + creds := b.GarageGlobalBucketS3APICredentials + + client, err := garage.NewS3APIClient(addr, creds) + if err != nil { + return nil, fmt.Errorf("connecting to garage S3 API At %q: %w", addr, err) + } + + return client, err +} diff --git a/go-workspace/src/cmd/entrypoint/daemon.go b/go-workspace/src/cmd/entrypoint/daemon.go index 7dc1122..2f1702a 100644 --- a/go-workspace/src/cmd/entrypoint/daemon.go +++ b/go-workspace/src/cmd/entrypoint/daemon.go @@ -36,6 +36,9 @@ import ( // * Merges the user-provided daemon.yml file with the default, and writes the // result to the runtime dir. // +// * Merges daemon.yml configuration into the bootstrap configuration, and +// rewrites the bootstrap file. +// // * Sets up environment variables that all other sub-processes then use, based // on the runtime dir. // @@ -79,7 +82,7 @@ func writeDaemonYml(userDaemonYmlPath, builtinDaemonYmlPath, runtimeDirPath stri return nil } -func writeBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error { +func copyBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error { path := env.DataDirBootstrapPath() dirPath := filepath.Dir(path) @@ -97,7 +100,7 @@ func writeBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error { f.Close() if err != nil { - return fmt.Errorf("writing new bootstrap file to %q: %w", path, err) + return fmt.Errorf("copying bootstrap file to %q: %w", path, err) } if err := env.LoadBootstrap(path); err != nil { @@ -133,11 +136,11 @@ func reloadBootstrap(env *crypticnet.Env, s3Client garage.S3APIClient) (bool, er buf := new(bytes.Buffer) if err := env.Bootstrap.WithHosts(newHosts).WriteTo(buf); err != nil { - return false, fmt.Errorf("writing new bootstrap file: %w", err) + return false, fmt.Errorf("writing new bootstrap file to buffer: %w", err) } - if err := writeBootstrapToDataDir(env, buf); err != nil { - return false, fmt.Errorf("writing new bootstrap file: %w", err) + if err := copyBootstrapToDataDir(env, buf); err != nil { + return false, fmt.Errorf("copying new bootstrap file to data dir: %w", err) } return true, nil @@ -281,7 +284,7 @@ var subCmdDaemon = subCmd{ env := subCmdCtx.env - s3Client, err := env.GlobalBucketS3APIClient() + s3Client, err := env.Bootstrap.GlobalBucketS3APIClient() if err != nil { return fmt.Errorf("creating client for global bucket: %w", err) } @@ -349,7 +352,7 @@ var subCmdDaemon = subCmd{ return fmt.Errorf("opening file %q: %w", env.BootstrapPath, err) } - err = writeBootstrapToDataDir(env, f) + err = copyBootstrapToDataDir(env, f) f.Close() if err != nil { @@ -361,6 +364,46 @@ var subCmdDaemon = subCmd{ return fmt.Errorf("generating daemon.yml file: %w", err) } + { + // we update this Host's data using whatever configuration has been + // provided by daemon.yml. This way the daemon has the most + // up-to-date possible bootstrap. This updated bootstrap will later + // get updated in garage using update-global-bucket, so other hosts + // will see it as well. + + // ThisDaemon can only be called after writeDaemonYml. + daemon := env.ThisDaemon() + host := env.Bootstrap.ThisHost() + + host.Nebula.PublicAddr = daemon.VPN.PublicAddr + + host.Garage = nil + + if allocs := daemon.Storage.Allocations; len(allocs) > 0 { + + host.Garage = new(bootstrap.GarageHost) + + for _, alloc := range allocs { + host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{ + RPCPort: alloc.RPCPort, + S3APIPort: alloc.S3APIPort, + WebPort: alloc.WebPort, + }) + } + } + + env.Bootstrap.Hosts[host.Name] = host + + buf := new(bytes.Buffer) + if err := env.Bootstrap.WithHosts(env.Bootstrap.Hosts).WriteTo(buf); err != nil { + return fmt.Errorf("writing new bootstrap file to buffer: %w", err) + } + + if err := copyBootstrapToDataDir(env, buf); err != nil { + return fmt.Errorf("copying new bootstrap file to data dir: %w", err) + } + } + for key, val := range env.ToMap() { if err := os.Setenv(key, val); err != nil { return fmt.Errorf("failed to set %q to %q: %w", key, val, err) diff --git a/go-workspace/src/cmd/entrypoint/garage.go b/go-workspace/src/cmd/entrypoint/garage.go index 1365d8a..e516810 100644 --- a/go-workspace/src/cmd/entrypoint/garage.go +++ b/go-workspace/src/cmd/entrypoint/garage.go @@ -30,7 +30,7 @@ var subCmdGarageMC = subCmd{ env := subCmdCtx.env - s3APIAddr := env.ChooseGaragePeer().S3APIAddr() + s3APIAddr := env.Bootstrap.ChooseGaragePeer().S3APIAddr() if *keyID == "" || *keySecret == "" { @@ -90,7 +90,7 @@ var subCmdGarageCLI = subCmd{ args = append([]string{"garage"}, subCmdCtx.args...) cliEnv = append( os.Environ(), - "GARAGE_RPC_HOST="+env.ChooseGaragePeer().RPCAddr(), + "GARAGE_RPC_HOST="+env.Bootstrap.ChooseGaragePeer().RPCAddr(), "GARAGE_RPC_SECRET="+env.Bootstrap.GarageRPCSecret, ) ) diff --git a/go-workspace/src/cmd/entrypoint/hosts.go b/go-workspace/src/cmd/entrypoint/hosts.go index 6ae8cdb..d0c81fd 100644 --- a/go-workspace/src/cmd/entrypoint/hosts.go +++ b/go-workspace/src/cmd/entrypoint/hosts.go @@ -64,7 +64,7 @@ var subCmdHostsAdd = subCmd{ env := subCmdCtx.env - client, err := env.GlobalBucketS3APIClient() + client, err := env.Bootstrap.GlobalBucketS3APIClient() if err != nil { return fmt.Errorf("creating client for global bucket: %w", err) } @@ -88,7 +88,7 @@ var subCmdHostsList = subCmd{ env := subCmdCtx.env - client, err := env.GlobalBucketS3APIClient() + client, err := env.Bootstrap.GlobalBucketS3APIClient() if err != nil { return fmt.Errorf("creating client for global bucket: %w", err) } @@ -132,7 +132,7 @@ var subCmdHostsDelete = subCmd{ env := subCmdCtx.env - client, err := env.GlobalBucketS3APIClient() + client, err := env.Bootstrap.GlobalBucketS3APIClient() if err != nil { return fmt.Errorf("creating client for global bucket: %w", err) } @@ -195,7 +195,7 @@ var subCmdHostsMakeBootstrap = subCmd{ return fmt.Errorf("reading admin.tgz with --admin-path of %q: %w", *adminPath, err) } - client, err := env.GlobalBucketS3APIClient() + client, err := env.Bootstrap.GlobalBucketS3APIClient() if err != nil { return fmt.Errorf("creating client for global bucket: %w", err) } diff --git a/go-workspace/src/cmd/update-global-bucket/main.go b/go-workspace/src/cmd/update-global-bucket/main.go index 8729495..fb67cba 100644 --- a/go-workspace/src/cmd/update-global-bucket/main.go +++ b/go-workspace/src/cmd/update-global-bucket/main.go @@ -13,42 +13,18 @@ func Main() { log.Fatalf("reading envvars: %v", err) } - client, err := env.GlobalBucketS3APIClient() + client, err := env.Bootstrap.GlobalBucketS3APIClient() if err != nil { log.Fatalf("creating client for global bucket: %v", err) } - host := env.Bootstrap.ThisHost() + err = bootstrap.PutGarageBoostrapHost( + env.Context, + client, + env.Bootstrap.ThisHost(), + ) - // We update the Host for this host in place, prior to writing it via the - // bootstrap method. We want to ensure that any changes made via daemon are - // reflected into the bootstrap data which is pushed up. - // - // TODO it'd be better if this was done within the daemon command itself, - // prior to any sub-processes being started. This would help us avoid this - // weird logic here, and would prevent all sub-processes from needing to be - // restarted the first time the daemon is started after daemon.yml is - // modified. - daemon := env.ThisDaemon() - - host.Nebula.PublicAddr = daemon.VPN.PublicAddr - - host.Garage = nil - - if allocs := daemon.Storage.Allocations; len(allocs) > 0 { - - host.Garage = new(bootstrap.GarageHost) - - for _, alloc := range allocs { - host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{ - RPCPort: alloc.RPCPort, - S3APIPort: alloc.S3APIPort, - WebPort: alloc.WebPort, - }) - } - } - - if err := bootstrap.PutGarageBoostrapHost(env.Context, client, host); err != nil { + if err != nil { log.Fatal(err) } } diff --git a/go-workspace/src/garage.go b/go-workspace/src/garage.go deleted file mode 100644 index df4818a..0000000 --- a/go-workspace/src/garage.go +++ /dev/null @@ -1,44 +0,0 @@ -package crypticnet - -import ( - "cryptic-net/garage" - "fmt" -) - -// ChooseGaragePeer returns a Peer for a garage instance from the network. It -// will prefer a garage instance on this particular host, if there is one, but -// will otherwise return a random endpoint. -func (env *Env) ChooseGaragePeer() garage.Peer { - - // TODO this only works well within the daemon process, otherwise daemon.yml - // isn't available. - if allocs := env.ThisDaemon().Storage.Allocations; len(allocs) > 0 { - - return garage.Peer{ - IP: env.Bootstrap.ThisHost().Nebula.IP, - RPCPort: allocs[0].RPCPort, - S3APIPort: allocs[0].S3APIPort, - } - } - - for _, peer := range env.Bootstrap.GaragePeers() { - return peer - } - - panic("no garage instances configured") -} - -// GlobalBucketS3APIClient returns an S3 client pre-configured with access to -// the global bucket. -func (env *Env) GlobalBucketS3APIClient() (garage.S3APIClient, error) { - - addr := env.ChooseGaragePeer().S3APIAddr() - creds := env.Bootstrap.GarageGlobalBucketS3APICredentials - - client, err := garage.NewS3APIClient(addr, creds) - if err != nil { - return nil, fmt.Errorf("connecting to garage S3 API At %q: %w", addr, err) - } - - return client, err -}