micropelago/isle
4
1
Fork 0
Code Issues 1 Pull Requests Projects Releases 6 Activity

Update host's config in bootstrap using daemon.yml prior to starting pmux

Browse Source 
Previously if the `daemon.yml` of a host was changed it would first
start up, load that new daemon.yml in, persist the new configuration for
the host to garage using `update-garage-host`, pull that config back
down and persist it to the bootstrap in `runDaemonPmuxOnce`, and restart
all child processes so they get the new config.

Now, once `daemon.yml` is loaded in we immediately merge it into and
persist this host's bootstrap file, prior to ever starting child
processes. This removes the necessity of restarting those process at
start.

This change also allows the bootstrap file to be the sole repository of
information required to pick a garage node to connect to, since it is
presumably always as up-to-date as it can possibly be. This allows for
removing some more logic from `Env`.
This commit is contained in:
Brian Picciano 2022-10-16 15:38:15 +02:00
parent 51b2fbba36
commit 93bdd3ebd4
6 changed files with 102 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
go-workspace/src/bootstrap/garage.go
View File

@ -2,6 +2,7 @@ package bootstrap
import (
"cryptic-net/garage"
"fmt"
)
// Paths within the bootstrap FS related to garage.
@ -45,3 +46,41 @@ func (b Bootstrap) GarageRPCPeerAddrs() []string {
}
return addrs
}
// ChooseGaragePeer returns a Peer for a garage instance from the network. It
// will prefer a garage instance on this particular host, if there is one, but
// will otherwise return a random endpoint.
func (b Bootstrap) ChooseGaragePeer() garage.Peer {
thisHost := b.ThisHost()
if thisHost.Garage != nil && len(thisHost.Garage.Instances) > 0 {
inst := thisHost.Garage.Instances[0]
return garage.Peer{
IP: thisHost.Nebula.IP,
RPCPort: inst.RPCPort,
S3APIPort: inst.S3APIPort,
}
}
for _, peer := range b.GaragePeers() {
return peer
}
panic("no garage instances configured")
}
// GlobalBucketS3APIClient returns an S3 client pre-configured with access to
// the global bucket.
func (b Bootstrap) GlobalBucketS3APIClient() (garage.S3APIClient, error) {
addr := b.ChooseGaragePeer().S3APIAddr()
creds := b.GarageGlobalBucketS3APICredentials
client, err := garage.NewS3APIClient(addr, creds)
if err != nil {
return nil, fmt.Errorf("connecting to garage S3 API At %q: %w", addr, err)
}
return client, err
}

57
go-workspace/src/cmd/entrypoint/daemon.go
View File

@ -36,6 +36,9 @@ import (
// * Merges the user-provided daemon.yml file with the default, and writes the
// result to the runtime dir.
//
// * Merges daemon.yml configuration into the bootstrap configuration, and
// rewrites the bootstrap file.
//
// * Sets up environment variables that all other sub-processes then use, based
// on the runtime dir.
//
@ -79,7 +82,7 @@ func writeDaemonYml(userDaemonYmlPath, builtinDaemonYmlPath, runtimeDirPath stri
return nil
}
func writeBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
func copyBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
path := env.DataDirBootstrapPath()
dirPath := filepath.Dir(path)
@ -97,7 +100,7 @@ func writeBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
f.Close()
if err != nil {
return fmt.Errorf("writing new bootstrap file to %q: %w", path, err)
return fmt.Errorf("copying bootstrap file to %q: %w", path, err)
}
if err := env.LoadBootstrap(path); err != nil {
@ -133,11 +136,11 @@ func reloadBootstrap(env *crypticnet.Env, s3Client garage.S3APIClient) (bool, er
buf := new(bytes.Buffer)
if err := env.Bootstrap.WithHosts(newHosts).WriteTo(buf); err != nil {
return false, fmt.Errorf("writing new bootstrap file: %w", err)
return false, fmt.Errorf("writing new bootstrap file to buffer: %w", err)
}
if err := writeBootstrapToDataDir(env, buf); err != nil {
return false, fmt.Errorf("writing new bootstrap file: %w", err)
if err := copyBootstrapToDataDir(env, buf); err != nil {
return false, fmt.Errorf("copying new bootstrap file to data dir: %w", err)
}
return true, nil
@ -281,7 +284,7 @@ var subCmdDaemon = subCmd{
env := subCmdCtx.env
s3Client, err := env.GlobalBucketS3APIClient()
s3Client, err := env.Bootstrap.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
@ -349,7 +352,7 @@ var subCmdDaemon = subCmd{
return fmt.Errorf("opening file %q: %w", env.BootstrapPath, err)
}
err = writeBootstrapToDataDir(env, f)
err = copyBootstrapToDataDir(env, f)
f.Close()
if err != nil {
@ -361,6 +364,46 @@ var subCmdDaemon = subCmd{
return fmt.Errorf("generating daemon.yml file: %w", err)
}
{
// we update this Host's data using whatever configuration has been
// provided by daemon.yml. This way the daemon has the most
// up-to-date possible bootstrap. This updated bootstrap will later
// get updated in garage using update-global-bucket, so other hosts
// will see it as well.
// ThisDaemon can only be called after writeDaemonYml.
daemon := env.ThisDaemon()
host := env.Bootstrap.ThisHost()
host.Nebula.PublicAddr = daemon.VPN.PublicAddr
host.Garage = nil
if allocs := daemon.Storage.Allocations; len(allocs) > 0 {
host.Garage = new(bootstrap.GarageHost)
for _, alloc := range allocs {
host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
WebPort: alloc.WebPort,
})
}
}
env.Bootstrap.Hosts[host.Name] = host
buf := new(bytes.Buffer)
if err := env.Bootstrap.WithHosts(env.Bootstrap.Hosts).WriteTo(buf); err != nil {
return fmt.Errorf("writing new bootstrap file to buffer: %w", err)
}
if err := copyBootstrapToDataDir(env, buf); err != nil {
return fmt.Errorf("copying new bootstrap file to data dir: %w", err)
}
}
for key, val := range env.ToMap() {
if err := os.Setenv(key, val); err != nil {
return fmt.Errorf("failed to set %q to %q: %w", key, val, err)

4
go-workspace/src/cmd/entrypoint/garage.go
View File

@ -30,7 +30,7 @@ var subCmdGarageMC = subCmd{
env := subCmdCtx.env
s3APIAddr := env.ChooseGaragePeer().S3APIAddr()
s3APIAddr := env.Bootstrap.ChooseGaragePeer().S3APIAddr()
if *keyID == "" || *keySecret == "" {
@ -90,7 +90,7 @@ var subCmdGarageCLI = subCmd{
args = append([]string{"garage"}, subCmdCtx.args...)
cliEnv = append(
os.Environ(),
"GARAGE_RPC_HOST="+env.ChooseGaragePeer().RPCAddr(),
"GARAGE_RPC_HOST="+env.Bootstrap.ChooseGaragePeer().RPCAddr(),
"GARAGE_RPC_SECRET="+env.Bootstrap.GarageRPCSecret,
)
)

8
go-workspace/src/cmd/entrypoint/hosts.go
View File

@ -64,7 +64,7 @@ var subCmdHostsAdd = subCmd{
env := subCmdCtx.env
client, err := env.GlobalBucketS3APIClient()
client, err := env.Bootstrap.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
@ -88,7 +88,7 @@ var subCmdHostsList = subCmd{
env := subCmdCtx.env
client, err := env.GlobalBucketS3APIClient()
client, err := env.Bootstrap.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
@ -132,7 +132,7 @@ var subCmdHostsDelete = subCmd{
env := subCmdCtx.env
client, err := env.GlobalBucketS3APIClient()
client, err := env.Bootstrap.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
@ -195,7 +195,7 @@ var subCmdHostsMakeBootstrap = subCmd{
return fmt.Errorf("reading admin.tgz with --admin-path of %q: %w", *adminPath, err)
}
client, err := env.GlobalBucketS3APIClient()
client, err := env.Bootstrap.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}

38
go-workspace/src/cmd/update-global-bucket/main.go
View File

@ -13,42 +13,18 @@ func Main() {
log.Fatalf("reading envvars: %v", err)
}
client, err := env.GlobalBucketS3APIClient()
client, err := env.Bootstrap.GlobalBucketS3APIClient()
if err != nil {
log.Fatalf("creating client for global bucket: %v", err)
}
host := env.Bootstrap.ThisHost()
err = bootstrap.PutGarageBoostrapHost(
env.Context,
client,
env.Bootstrap.ThisHost(),
)
// We update the Host for this host in place, prior to writing it via the
// bootstrap method. We want to ensure that any changes made via daemon are
// reflected into the bootstrap data which is pushed up.
//
// TODO it'd be better if this was done within the daemon command itself,
// prior to any sub-processes being started. This would help us avoid this
// weird logic here, and would prevent all sub-processes from needing to be
// restarted the first time the daemon is started after daemon.yml is
// modified.
daemon := env.ThisDaemon()
host.Nebula.PublicAddr = daemon.VPN.PublicAddr
host.Garage = nil
if allocs := daemon.Storage.Allocations; len(allocs) > 0 {
host.Garage = new(bootstrap.GarageHost)
for _, alloc := range allocs {
host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
WebPort: alloc.WebPort,
})
}
}
if err := bootstrap.PutGarageBoostrapHost(env.Context, client, host); err != nil {
if err != nil {
log.Fatal(err)
}
}

44
go-workspace/src/garage.go
View File

@ -1,44 +0,0 @@
package crypticnet
import (
"cryptic-net/garage"
"fmt"
)
// ChooseGaragePeer returns a Peer for a garage instance from the network. It
// will prefer a garage instance on this particular host, if there is one, but
// will otherwise return a random endpoint.
func (env *Env) ChooseGaragePeer() garage.Peer {
// TODO this only works well within the daemon process, otherwise daemon.yml
// isn't available.
if allocs := env.ThisDaemon().Storage.Allocations; len(allocs) > 0 {
return garage.Peer{
IP: env.Bootstrap.ThisHost().Nebula.IP,
RPCPort: allocs[0].RPCPort,
S3APIPort: allocs[0].S3APIPort,
}
}
for _, peer := range env.Bootstrap.GaragePeers() {
return peer
}
panic("no garage instances configured")
}
// GlobalBucketS3APIClient returns an S3 client pre-configured with access to
// the global bucket.
func (env *Env) GlobalBucketS3APIClient() (garage.S3APIClient, error) {
addr := env.ChooseGaragePeer().S3APIAddr()
creds := env.Bootstrap.GarageGlobalBucketS3APICredentials
client, err := garage.NewS3APIClient(addr, creds)
if err != nil {
return nil, fmt.Errorf("connecting to garage S3 API At %q: %w", addr, err)
}
return client, err
}