diff --git a/go-workspace/src/bootstrap/garage.go b/go-workspace/src/bootstrap/garage.go index cdce6bd..6d33af4 100644 --- a/go-workspace/src/bootstrap/garage.go +++ b/go-workspace/src/bootstrap/garage.go @@ -2,6 +2,7 @@ package bootstrap import ( "cryptic-net/garage" + "fmt" ) // Paths within the bootstrap FS related to garage. @@ -45,3 +46,41 @@ func (b Bootstrap) GarageRPCPeerAddrs() []string { } return addrs } + +// ChooseGaragePeer returns a Peer for a garage instance from the network. It +// will prefer a garage instance on this particular host, if there is one, but +// will otherwise return a random endpoint. +func (b Bootstrap) ChooseGaragePeer() garage.Peer { + + thisHost := b.ThisHost() + + if thisHost.Garage != nil && len(thisHost.Garage.Instances) > 0 { + inst := thisHost.Garage.Instances[0] + return garage.Peer{ + IP: thisHost.Nebula.IP, + RPCPort: inst.RPCPort, + S3APIPort: inst.S3APIPort, + } + } + + for _, peer := range b.GaragePeers() { + return peer + } + + panic("no garage instances configured") +} + +// GlobalBucketS3APIClient returns an S3 client pre-configured with access to +// the global bucket. +func (b Bootstrap) GlobalBucketS3APIClient() (garage.S3APIClient, error) { + + addr := b.ChooseGaragePeer().S3APIAddr() + creds := b.GarageGlobalBucketS3APICredentials + + client, err := garage.NewS3APIClient(addr, creds) + if err != nil { + return nil, fmt.Errorf("connecting to garage S3 API At %q: %w", addr, err) + } + + return client, err +} diff --git a/go-workspace/src/cmd/entrypoint/daemon.go b/go-workspace/src/cmd/entrypoint/daemon.go index 7dc1122..2f1702a 100644 --- a/go-workspace/src/cmd/entrypoint/daemon.go +++ b/go-workspace/src/cmd/entrypoint/daemon.go @@ -36,6 +36,9 @@ import ( // * Merges the user-provided daemon.yml file with the default, and writes the // result to the runtime dir. // +// * Merges daemon.yml configuration into the bootstrap configuration, and +// rewrites the bootstrap file. +// // * Sets up environment variables that all other sub-processes then use, based // on the runtime dir. // @@ -79,7 +82,7 @@ func writeDaemonYml(userDaemonYmlPath, builtinDaemonYmlPath, runtimeDirPath stri return nil } -func writeBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error { +func copyBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error { path := env.DataDirBootstrapPath() dirPath := filepath.Dir(path) @@ -97,7 +100,7 @@ func writeBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error { f.Close() if err != nil { - return fmt.Errorf("writing new bootstrap file to %q: %w", path, err) + return fmt.Errorf("copying bootstrap file to %q: %w", path, err) } if err := env.LoadBootstrap(path); err != nil { @@ -133,11 +136,11 @@ func reloadBootstrap(env *crypticnet.Env, s3Client garage.S3APIClient) (bool, er buf := new(bytes.Buffer) if err := env.Bootstrap.WithHosts(newHosts).WriteTo(buf); err != nil { - return false, fmt.Errorf("writing new bootstrap file: %w", err) + return false, fmt.Errorf("writing new bootstrap file to buffer: %w", err) } - if err := writeBootstrapToDataDir(env, buf); err != nil { - return false, fmt.Errorf("writing new bootstrap file: %w", err) + if err := copyBootstrapToDataDir(env, buf); err != nil { + return false, fmt.Errorf("copying new bootstrap file to data dir: %w", err) } return true, nil @@ -281,7 +284,7 @@ var subCmdDaemon = subCmd{ env := subCmdCtx.env - s3Client, err := env.GlobalBucketS3APIClient() + s3Client, err := env.Bootstrap.GlobalBucketS3APIClient() if err != nil { return fmt.Errorf("creating client for global bucket: %w", err) } @@ -349,7 +352,7 @@ var subCmdDaemon = subCmd{ return fmt.Errorf("opening file %q: %w", env.BootstrapPath, err) } - err = writeBootstrapToDataDir(env, f) + err = copyBootstrapToDataDir(env, f) f.Close() if err != nil { @@ -361,6 +364,46 @@ var subCmdDaemon = subCmd{ return fmt.Errorf("generating daemon.yml file: %w", err) } + { + // we update this Host's data using whatever configuration has been + // provided by daemon.yml. This way the daemon has the most + // up-to-date possible bootstrap. This updated bootstrap will later + // get updated in garage using update-global-bucket, so other hosts + // will see it as well. + + // ThisDaemon can only be called after writeDaemonYml. + daemon := env.ThisDaemon() + host := env.Bootstrap.ThisHost() + + host.Nebula.PublicAddr = daemon.VPN.PublicAddr + + host.Garage = nil + + if allocs := daemon.Storage.Allocations; len(allocs) > 0 { + + host.Garage = new(bootstrap.GarageHost) + + for _, alloc := range allocs { + host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{ + RPCPort: alloc.RPCPort, + S3APIPort: alloc.S3APIPort, + WebPort: alloc.WebPort, + }) + } + } + + env.Bootstrap.Hosts[host.Name] = host + + buf := new(bytes.Buffer) + if err := env.Bootstrap.WithHosts(env.Bootstrap.Hosts).WriteTo(buf); err != nil { + return fmt.Errorf("writing new bootstrap file to buffer: %w", err) + } + + if err := copyBootstrapToDataDir(env, buf); err != nil { + return fmt.Errorf("copying new bootstrap file to data dir: %w", err) + } + } + for key, val := range env.ToMap() { if err := os.Setenv(key, val); err != nil { return fmt.Errorf("failed to set %q to %q: %w", key, val, err) diff --git a/go-workspace/src/cmd/entrypoint/garage.go b/go-workspace/src/cmd/entrypoint/garage.go index 1365d8a..e516810 100644 --- a/go-workspace/src/cmd/entrypoint/garage.go +++ b/go-workspace/src/cmd/entrypoint/garage.go @@ -30,7 +30,7 @@ var subCmdGarageMC = subCmd{ env := subCmdCtx.env - s3APIAddr := env.ChooseGaragePeer().S3APIAddr() + s3APIAddr := env.Bootstrap.ChooseGaragePeer().S3APIAddr() if *keyID == "" || *keySecret == "" { @@ -90,7 +90,7 @@ var subCmdGarageCLI = subCmd{ args = append([]string{"garage"}, subCmdCtx.args...) cliEnv = append( os.Environ(), - "GARAGE_RPC_HOST="+env.ChooseGaragePeer().RPCAddr(), + "GARAGE_RPC_HOST="+env.Bootstrap.ChooseGaragePeer().RPCAddr(), "GARAGE_RPC_SECRET="+env.Bootstrap.GarageRPCSecret, ) ) diff --git a/go-workspace/src/cmd/entrypoint/hosts.go b/go-workspace/src/cmd/entrypoint/hosts.go index 6ae8cdb..d0c81fd 100644 --- a/go-workspace/src/cmd/entrypoint/hosts.go +++ b/go-workspace/src/cmd/entrypoint/hosts.go @@ -64,7 +64,7 @@ var subCmdHostsAdd = subCmd{ env := subCmdCtx.env - client, err := env.GlobalBucketS3APIClient() + client, err := env.Bootstrap.GlobalBucketS3APIClient() if err != nil { return fmt.Errorf("creating client for global bucket: %w", err) } @@ -88,7 +88,7 @@ var subCmdHostsList = subCmd{ env := subCmdCtx.env - client, err := env.GlobalBucketS3APIClient() + client, err := env.Bootstrap.GlobalBucketS3APIClient() if err != nil { return fmt.Errorf("creating client for global bucket: %w", err) } @@ -132,7 +132,7 @@ var subCmdHostsDelete = subCmd{ env := subCmdCtx.env - client, err := env.GlobalBucketS3APIClient() + client, err := env.Bootstrap.GlobalBucketS3APIClient() if err != nil { return fmt.Errorf("creating client for global bucket: %w", err) } @@ -195,7 +195,7 @@ var subCmdHostsMakeBootstrap = subCmd{ return fmt.Errorf("reading admin.tgz with --admin-path of %q: %w", *adminPath, err) } - client, err := env.GlobalBucketS3APIClient() + client, err := env.Bootstrap.GlobalBucketS3APIClient() if err != nil { return fmt.Errorf("creating client for global bucket: %w", err) } diff --git a/go-workspace/src/cmd/update-global-bucket/main.go b/go-workspace/src/cmd/update-global-bucket/main.go index 8729495..fb67cba 100644 --- a/go-workspace/src/cmd/update-global-bucket/main.go +++ b/go-workspace/src/cmd/update-global-bucket/main.go @@ -13,42 +13,18 @@ func Main() { log.Fatalf("reading envvars: %v", err) } - client, err := env.GlobalBucketS3APIClient() + client, err := env.Bootstrap.GlobalBucketS3APIClient() if err != nil { log.Fatalf("creating client for global bucket: %v", err) } - host := env.Bootstrap.ThisHost() + err = bootstrap.PutGarageBoostrapHost( + env.Context, + client, + env.Bootstrap.ThisHost(), + ) - // We update the Host for this host in place, prior to writing it via the - // bootstrap method. We want to ensure that any changes made via daemon are - // reflected into the bootstrap data which is pushed up. - // - // TODO it'd be better if this was done within the daemon command itself, - // prior to any sub-processes being started. This would help us avoid this - // weird logic here, and would prevent all sub-processes from needing to be - // restarted the first time the daemon is started after daemon.yml is - // modified. - daemon := env.ThisDaemon() - - host.Nebula.PublicAddr = daemon.VPN.PublicAddr - - host.Garage = nil - - if allocs := daemon.Storage.Allocations; len(allocs) > 0 { - - host.Garage = new(bootstrap.GarageHost) - - for _, alloc := range allocs { - host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{ - RPCPort: alloc.RPCPort, - S3APIPort: alloc.S3APIPort, - WebPort: alloc.WebPort, - }) - } - } - - if err := bootstrap.PutGarageBoostrapHost(env.Context, client, host); err != nil { + if err != nil { log.Fatal(err) } } diff --git a/go-workspace/src/garage.go b/go-workspace/src/garage.go deleted file mode 100644 index df4818a..0000000 --- a/go-workspace/src/garage.go +++ /dev/null @@ -1,44 +0,0 @@ -package crypticnet - -import ( - "cryptic-net/garage" - "fmt" -) - -// ChooseGaragePeer returns a Peer for a garage instance from the network. It -// will prefer a garage instance on this particular host, if there is one, but -// will otherwise return a random endpoint. -func (env *Env) ChooseGaragePeer() garage.Peer { - - // TODO this only works well within the daemon process, otherwise daemon.yml - // isn't available. - if allocs := env.ThisDaemon().Storage.Allocations; len(allocs) > 0 { - - return garage.Peer{ - IP: env.Bootstrap.ThisHost().Nebula.IP, - RPCPort: allocs[0].RPCPort, - S3APIPort: allocs[0].S3APIPort, - } - } - - for _, peer := range env.Bootstrap.GaragePeers() { - return peer - } - - panic("no garage instances configured") -} - -// GlobalBucketS3APIClient returns an S3 client pre-configured with access to -// the global bucket. -func (env *Env) GlobalBucketS3APIClient() (garage.S3APIClient, error) { - - addr := env.ChooseGaragePeer().S3APIAddr() - creds := env.Bootstrap.GarageGlobalBucketS3APICredentials - - client, err := garage.NewS3APIClient(addr, creds) - if err != nil { - return nil, fmt.Errorf("connecting to garage S3 API At %q: %w", addr, err) - } - - return client, err -}