diff --git a/go-workspace/src/bootstrap/bootstrap.go b/go-workspace/src/bootstrap/bootstrap.go index 11cb80e..8f49696 100644 --- a/go-workspace/src/bootstrap/bootstrap.go +++ b/go-workspace/src/bootstrap/bootstrap.go @@ -5,16 +5,21 @@ import ( "cryptic-net/garage" "cryptic-net/tarutil" "cryptic-net/yamlutil" + "crypto/sha512" "fmt" "io" "io/fs" "os" + "path/filepath" + "sort" "strings" + + "gopkg.in/yaml.v3" ) // Paths within the bootstrap FS which for general data. const ( - HostNamePath = "hostname" + hostNamePath = "hostname" ) // Bootstrap is used for accessing all information contained within a @@ -32,14 +37,6 @@ type Bootstrap struct { GarageRPCSecret string GarageGlobalBucketS3APICredentials garage.S3APICredentials - - // Hash is a determinstic hash of the contents of the bootstrap file. This - // will be populated when parsing a Bootstrap from a bootstrap.tgz, but will - // be ignored when creating a new bootstrap.tgz. - Hash []byte - - // DEPRECATED do not use - FS fs.FS } // FromFS loads a Boostrap instance from the given fs.FS, which presumably @@ -51,8 +48,6 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) { err error ) - b.FS = bootstrapFS - if b.Hosts, err = loadHosts(bootstrapFS); err != nil { return Bootstrap{}, fmt.Errorf("loading hosts info from fs: %w", err) } @@ -60,7 +55,7 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) { if err = yamlutil.LoadYamlFSFile( &b.GarageGlobalBucketS3APICredentials, bootstrapFS, - GarageGlobalBucketKeyYmlPath, + garageGlobalBucketKeyYmlPath, ); err != nil { return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", b.GarageGlobalBucketS3APICredentials, err) } @@ -69,11 +64,11 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) { into *string path string }{ - {&b.HostName, HostNamePath}, - {&b.NebulaCertsCACert, NebulaCertsCACertPath}, - {&b.NebulaCertsHostCert, NebulaCertsHostCertPath}, - {&b.NebulaCertsHostKey, NebulaCertsHostKeyPath}, - {&b.GarageRPCSecret, GarageRPCSecretPath}, + {&b.HostName, hostNamePath}, + {&b.NebulaCertsCACert, nebulaCertsCACertPath}, + {&b.NebulaCertsHostCert, nebulaCertsHostCertPath}, + {&b.NebulaCertsHostKey, nebulaCertsHostKeyPath}, + {&b.GarageRPCSecret, garageRPCSecretPath}, } for _, f := range filesToLoadAsString { @@ -87,10 +82,6 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) { // TODO confirm if this is necessary b.GarageRPCSecret = strings.TrimSpace(b.GarageRPCSecret) - if b.Hash, err = fs.ReadFile(bootstrapFS, tarutil.HashBinPath); err != nil { - return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", tarutil.HashBinPath, err) - } - return b, nil } @@ -117,6 +108,48 @@ func FromFile(path string) (Bootstrap, error) { return FromReader(f) } +// WriteTo writes the Bootstrap as a new bootstrap.tgz to the given io.Writer. +func (b Bootstrap) WriteTo(into io.Writer) error { + + w := tarutil.NewTGZWriter(into) + + filesToWriteAsString := []struct { + value string + path string + }{ + {b.HostName, hostNamePath}, + {b.NebulaCertsCACert, nebulaCertsCACertPath}, + {b.NebulaCertsHostCert, nebulaCertsHostCertPath}, + {b.NebulaCertsHostKey, nebulaCertsHostKeyPath}, + {b.GarageRPCSecret, garageRPCSecretPath}, + } + + for _, f := range filesToWriteAsString { + w.WriteFileBytes(f.path, []byte(f.value)) + } + + garageGlobalBucketKeyB, err := yaml.Marshal(b.GarageGlobalBucketS3APICredentials) + if err != nil { + return fmt.Errorf("yaml encoding garage global bucket creds: %w", err) + } + + w.WriteFileBytes(garageGlobalBucketKeyYmlPath, garageGlobalBucketKeyB) + + for _, host := range b.Hosts { + + hostB, err := yaml.Marshal(host) + if err != nil { + return fmt.Errorf("yaml encoding host %#v: %w", host, err) + } + + path := filepath.Join(hostsDirPath, host.Name+".yml") + + w.WriteFileBytes(path, hostB) + } + + return w.Close() +} + // ThisHost is a shortcut for b.Hosts[b.HostName], but will panic if the // HostName isn't found in the Hosts map. func (b Bootstrap) ThisHost() Host { @@ -128,3 +161,38 @@ func (b Bootstrap) ThisHost() Host { return host } + +// Hash returns a deterministic hash of the given hosts map. +func HostsHash(hostsMap map[string]Host) ([]byte, error) { + + hosts := make([]Host, 0, len(hostsMap)) + for _, host := range hostsMap { + hosts = append(hosts, host) + } + + sort.Slice(hosts, func(i, j int) bool { return hosts[i].Name < hosts[j].Name }) + + h := sha512.New() + + if err := yaml.NewEncoder(h).Encode(hosts); err != nil { + return nil, err + } + + return h.Sum(nil), nil +} + +// WithHosts returns a copy of the Bootstrap with the given set of Hosts applied +// to it. It will _not_ overwrite the Host for _this_ host, however. +func (b Bootstrap) WithHosts(hosts map[string]Host) Bootstrap { + + hostsCopy := make(map[string]Host, len(hosts)) + + for name, host := range hosts { + hostsCopy[name] = host + } + + hostsCopy[b.HostName] = b.ThisHost() + + b.Hosts = hostsCopy + return b +} diff --git a/go-workspace/src/bootstrap/creator/creator.go b/go-workspace/src/bootstrap/creator/creator.go deleted file mode 100644 index 6807818..0000000 --- a/go-workspace/src/bootstrap/creator/creator.go +++ /dev/null @@ -1,110 +0,0 @@ -// Package creator is responsible for creating bootstrap files. It exists -// separately from the main bootstrap package in order to prevent import loops -// due to its use of crypticnet.Env. -package creator - -import ( - "context" - crypticnet "cryptic-net" - "cryptic-net/bootstrap" - "cryptic-net/nebula" - "cryptic-net/tarutil" - "fmt" - "io" - "io/fs" -) - -func newBootstrap( - ctx context.Context, - into io.Writer, - hostname provider, - nebulaCertsCACert provider, - nebulaCertsHostCert provider, - nebulaCertsHostKey provider, - nebulaHosts provider, - garageRPCSecret provider, - garageGlobalBucketKey provider, - garageHosts provider, -) error { - - pairs := []struct { - path string - provider provider - }{ - {bootstrap.HostNamePath, hostname}, - {bootstrap.NebulaCertsCACertPath, nebulaCertsCACert}, - {bootstrap.NebulaCertsHostCertPath, nebulaCertsHostCert}, - {bootstrap.NebulaCertsHostKeyPath, nebulaCertsHostKey}, - {bootstrap.NebulaHostsDirPath, nebulaHosts}, - {bootstrap.GarageRPCSecretPath, garageRPCSecret}, - {bootstrap.GarageGlobalBucketKeyYmlPath, garageGlobalBucketKey}, - {bootstrap.GarageHostsDirPath, garageHosts}, - } - - w := tarutil.NewTGZWriter(into) - - for _, pair := range pairs { - - if err := pair.provider(ctx, w, pair.path); err != nil { - return fmt.Errorf("populating %q in new bootstrap: %w", pair.path, err) - } - } - - return w.Close() -} - -// NewForThisHost generates a new bootstrap file for the current host, based on -// the existing environment as well as data in garage. -func NewForThisHost(env *crypticnet.Env, into io.Writer) error { - - client, err := env.GlobalBucketS3APIClient() - if err != nil { - return fmt.Errorf("creating client for global bucket: %w", err) - } - - return newBootstrap( - env.Context, - into, - provideFromFS(env.Bootstrap.FS), // hostname - provideFromFS(env.Bootstrap.FS), // nebulaCertsCACert - provideFromFS(env.Bootstrap.FS), // nebulaCertsHostCert - provideFromFS(env.Bootstrap.FS), // nebulaCertsHostKey - provideDirFromGarage(client), // nebulaHosts - provideFromFS(env.Bootstrap.FS), // garageRPCSecret - provideFromFS(env.Bootstrap.FS), // garageGlobalBucketKey - provideDirFromGarage(client), // garageHosts - ) -} - -// NewForHost generates a new bootstrap file for an arbitrary host, based on the -// given admin file's FS and data in garage. -func NewForHost(env *crypticnet.Env, adminFS fs.FS, name string, into io.Writer) error { - - host, ok := env.Bootstrap.Hosts[name] - if !ok { - return fmt.Errorf("unknown host %q, make sure host entry has been created", name) - } - - client, err := env.GlobalBucketS3APIClient() - if err != nil { - return fmt.Errorf("creating client for global bucket: %w", err) - } - - nebulaHostCert, err := nebula.NewHostCert(adminFS, host.Nebula) - if err != nil { - return fmt.Errorf("creating new nebula host key/cert: %w", err) - } - - return newBootstrap( - env.Context, - into, - provideFromBytes([]byte(name)), // hostname - provideFromBytes(nebulaHostCert.CACert), // nebulaCertsCACert - provideFromBytes(nebulaHostCert.HostCert), // nebulaCertsHostCert - provideFromBytes(nebulaHostCert.HostKey), // nebulaCertsHostKey - provideDirFromGarage(client), // nebulaHosts - provideFromFS(adminFS), // garageRPCSecret - provideFromFS(adminFS), // garageGlobalBucketKey - provideDirFromGarage(client), // garageHosts - ) -} diff --git a/go-workspace/src/bootstrap/creator/provider.go b/go-workspace/src/bootstrap/creator/provider.go deleted file mode 100644 index 4361172..0000000 --- a/go-workspace/src/bootstrap/creator/provider.go +++ /dev/null @@ -1,121 +0,0 @@ -package creator - -import ( - "context" - "cryptic-net/garage" - "cryptic-net/tarutil" - "fmt" - "io/fs" - - "github.com/minio/minio-go/v7" -) - -// provider is a function which will populate the given filePath into the given -// TGZWriter. The path may be a file or a directory. -type provider func(context.Context, *tarutil.TGZWriter, string) error - -func provideFromBytes(body []byte) provider { - - return func( - ctx context.Context, - w *tarutil.TGZWriter, - filePath string, - ) error { - - w.WriteFileBytes(filePath, body) - return nil - } -} - -func provideFromFS(srcFS fs.FS) provider { - - return func( - ctx context.Context, - w *tarutil.TGZWriter, - filePath string, - ) error { - - return w.CopyFileFromFS(filePath, srcFS) - } -} - -func provideDirFromFS(srcFS fs.FS) provider { - - return func( - ctx context.Context, - w *tarutil.TGZWriter, - dirPath string, - ) error { - - return fs.WalkDir( - srcFS, dirPath, - func(filePath string, dirEntry fs.DirEntry, err error) error { - - if err != nil { - return err - - } else if dirEntry.IsDir() { - return nil - - } else if err := w.CopyFileFromFS(filePath, srcFS); err != nil { - return fmt.Errorf("copying file %q: %w", filePath, err) - } - - return nil - }, - ) - - } -} - -// TODO it'd be great if we could wrap a minio.Client into an fs.FS. That would -// get rid of a weird dependency in this package, and clean up this code a ton. -func provideDirFromGarage(client *minio.Client) provider { - - return func( - ctx context.Context, - w *tarutil.TGZWriter, - dirPath string, - ) error { - - objInfoCh := client.ListObjects( - ctx, garage.GlobalBucket, - minio.ListObjectsOptions{ - Prefix: dirPath, - Recursive: true, - }, - ) - - for objInfo := range objInfoCh { - - if objInfo.Err != nil { - return fmt.Errorf("listing objects: %w", objInfo.Err) - } - - obj, err := client.GetObject( - ctx, garage.GlobalBucket, objInfo.Key, minio.GetObjectOptions{}, - ) - - if err != nil { - return fmt.Errorf( - "retrieving object %q from global bucket: %w", - objInfo.Key, err, - ) - } - - objStat, err := obj.Stat() - if err != nil { - obj.Close() - return fmt.Errorf( - "stating object %q from global bucket: %w", - objInfo.Key, err, - ) - } - - w.WriteFile(objInfo.Key, objStat.Size, obj) - obj.Close() - } - - return nil - } -} diff --git a/go-workspace/src/bootstrap/garage.go b/go-workspace/src/bootstrap/garage.go index 1be0e00..775a6ae 100644 --- a/go-workspace/src/bootstrap/garage.go +++ b/go-workspace/src/bootstrap/garage.go @@ -6,9 +6,8 @@ import ( // Paths within the bootstrap FS related to garage. const ( - GarageGlobalBucketKeyYmlPath = "garage/global-bucket-key.yml" - GarageRPCSecretPath = "garage/rpc-secret.txt" - GarageHostsDirPath = "garage/hosts" + garageGlobalBucketKeyYmlPath = "garage/global-bucket-key.yml" + garageRPCSecretPath = "garage/rpc-secret.txt" ) // GaragePeers returns a Peer for each known garage instance in the network. diff --git a/go-workspace/src/bootstrap/garage_global_bucket.go b/go-workspace/src/bootstrap/garage_global_bucket.go new file mode 100644 index 0000000..77a8e25 --- /dev/null +++ b/go-workspace/src/bootstrap/garage_global_bucket.go @@ -0,0 +1,111 @@ +package bootstrap + +import ( + "bytes" + "context" + "cryptic-net/garage" + "fmt" + "log" + "path/filepath" + + "github.com/minio/minio-go/v7" + "gopkg.in/yaml.v3" +) + +// Paths within garage's global bucket +const ( + garageGlobalBucketBootstrapHostsDirPath = "bootstrap/hosts" +) + +// PutGarageBoostrapHost places the .yml file for the given host into +// garage so that other hosts are able to see relevant configuration for it. +// +// The given client should be for the global bucket. +func PutGarageBoostrapHost( + ctx context.Context, client garage.S3APIClient, host Host, +) error { + + buf := new(bytes.Buffer) + + if err := yaml.NewEncoder(buf).Encode(host); err != nil { + log.Fatalf("yaml encoding host data: %v", err) + } + + filePath := filepath.Join(garageGlobalBucketBootstrapHostsDirPath, host.Name+".yml") + + _, err := client.PutObject( + ctx, garage.GlobalBucket, filePath, buf, int64(buf.Len()), + minio.PutObjectOptions{}, + ) + + if err != nil { + return fmt.Errorf("writing to %q in global bucket: %w", filePath, err) + } + + return nil +} + +// RemoveGarageBootstrapHost removes the .yml for the given host from +// garage. +// +// The given client should be for the global bucket. +func RemoveGarageBootstrapHost( + ctx context.Context, client garage.S3APIClient, hostName string, +) error { + + filePath := filepath.Join(garageGlobalBucketBootstrapHostsDirPath, hostName+".yml") + + return client.RemoveObject( + ctx, garage.GlobalBucket, filePath, + minio.RemoveObjectOptions{}, + ) +} + +// GetGarageBootstrapHosts loads the .yml file for all hosts stored in +// garage. +// +// The given client should be for the global bucket. +func GetGarageBootstrapHosts( + ctx context.Context, client garage.S3APIClient, +) ( + map[string]Host, error, +) { + + hosts := map[string]Host{} + + objInfoCh := client.ListObjects( + ctx, garage.GlobalBucket, + minio.ListObjectsOptions{ + Prefix: garageGlobalBucketBootstrapHostsDirPath, + Recursive: true, + }, + ) + + for objInfo := range objInfoCh { + + if objInfo.Err != nil { + return nil, fmt.Errorf("listing objects: %w", objInfo.Err) + } + + obj, err := client.GetObject( + ctx, garage.GlobalBucket, objInfo.Key, minio.GetObjectOptions{}, + ) + + if err != nil { + return nil, fmt.Errorf("retrieving object %q: %w", objInfo.Key, err) + } + + var host Host + + err = yaml.NewDecoder(obj).Decode(&host) + obj.Close() + + if err != nil { + return nil, fmt.Errorf("yaml decoding object %q: %w", objInfo.Key, err) + } + + hosts[host.Name] = host + } + + return hosts, nil +} diff --git a/go-workspace/src/bootstrap/hosts.go b/go-workspace/src/bootstrap/hosts.go index 1fbe79e..b15eef7 100644 --- a/go-workspace/src/bootstrap/hosts.go +++ b/go-workspace/src/bootstrap/hosts.go @@ -10,21 +10,26 @@ import ( "gopkg.in/yaml.v3" ) -// NebulaHost describes the contents of a `./nebula/hosts/.yml` file. +const ( + hostsDirPath = "hosts" +) + +// NebulaHost describes the nebula configuration of a Host which is relevant for +// other hosts to know. type NebulaHost struct { - Name string `yaml:"name"` IP string `yaml:"ip"` PublicAddr string `yaml:"public_addr,omitempty"` } -// GarageHostInstance describes a single garage instance running on a host. +// GarageHost describes a single garage instance in the GarageHost. type GarageHostInstance struct { RPCPort int `yaml:"rpc_port"` S3APIPort int `yaml:"s3_api_port"` WebPort int `yaml:"web_port"` } -// GarageHost describes the contents of a `./garage/hosts/.yml` file. +// GarageHost describes the garage configuration of a Host which is relevant for +// other hosts to know. type GarageHost struct { Instances []GarageHostInstance `yaml:"instances"` } @@ -32,12 +37,12 @@ type GarageHost struct { // Host consolidates all information about a single host from the bootstrap // file. type Host struct { - Name string - Nebula NebulaHost - Garage *GarageHost + Name string `yaml:"name"` + Nebula NebulaHost `yaml:"nebula"` + Garage *GarageHost `yaml:"garage,omitempty"` } -func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) { +func loadHostsLegacy(bootstrapFS fs.FS) (map[string]Host, error) { hosts := map[string]Host{} @@ -51,7 +56,7 @@ func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) { } { - globPath := filepath.Join(NebulaHostsDirPath, "*.yml") + globPath := "nebula/hosts/*.yml" nebulaHostFiles, err := fs.Glob(bootstrapFS, globPath) if err != nil { @@ -77,7 +82,7 @@ func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) { for hostName, host := range hosts { - garageHostPath := filepath.Join(GarageHostsDirPath, hostName+".yml") + garageHostPath := filepath.Join("garage/hosts", hostName+".yml") var garageHost GarageHost if err := readAsYaml(&garageHost, garageHostPath); errors.Is(err, fs.ErrNotExist) { @@ -92,3 +97,55 @@ func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) { return hosts, nil } + +func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) { + + hosts := map[string]Host{} + + readAsYaml := func(into interface{}, path string) error { + b, err := fs.ReadFile(bootstrapFS, path) + if err != nil { + return fmt.Errorf("reading file from fs: %w", err) + } + + return yaml.Unmarshal(b, into) + } + + globPath := filepath.Join(hostsDirPath, "*.yml") + + hostPaths, err := fs.Glob(bootstrapFS, globPath) + if err != nil { + return nil, fmt.Errorf("listing host files at %q in fs: %w", globPath, err) + } + + for _, hostPath := range hostPaths { + + hostName := filepath.Base(hostPath) + hostName = strings.TrimSuffix(hostName, filepath.Ext(hostName)) + + var host Host + if err := readAsYaml(&host, hostPath); err != nil { + return nil, fmt.Errorf("reading %q as yaml: %w", hostPath, err) + } + + hosts[hostName] = host + } + + if len(hosts) > 0 { + return hosts, nil + } + + // We used to have the bootstrap file laid out differently. If no hosts were + // found then the bootstrap file is probably in that format. + hosts, err = loadHostsLegacy(bootstrapFS) + + if err != nil { + return nil, fmt.Errorf("loading hosts in legacy layout from fs: %w", err) + } + + if len(hosts) == 0 { + return nil, fmt.Errorf("failed to load any hosts from fs") + } + + return hosts, nil +} diff --git a/go-workspace/src/bootstrap/nebula.go b/go-workspace/src/bootstrap/nebula.go index f2888a5..f641a95 100644 --- a/go-workspace/src/bootstrap/nebula.go +++ b/go-workspace/src/bootstrap/nebula.go @@ -2,9 +2,7 @@ package bootstrap // Paths within the bootstrap FS related to nebula. const ( - NebulaHostsDirPath = "nebula/hosts" - - NebulaCertsCACertPath = "nebula/certs/ca.crt" - NebulaCertsHostCertPath = "nebula/certs/host.crt" - NebulaCertsHostKeyPath = "nebula/certs/host.key" + nebulaCertsCACertPath = "nebula/certs/ca.crt" + nebulaCertsHostCertPath = "nebula/certs/host.crt" + nebulaCertsHostKeyPath = "nebula/certs/host.key" ) diff --git a/go-workspace/src/cmd/cryptic-net-main/main.go b/go-workspace/src/cmd/cryptic-net-main/main.go index f9b6905..613294b 100644 --- a/go-workspace/src/cmd/cryptic-net-main/main.go +++ b/go-workspace/src/cmd/cryptic-net-main/main.go @@ -19,9 +19,8 @@ import ( garage_entrypoint "cryptic-net/cmd/garage-entrypoint" garage_layout_diff "cryptic-net/cmd/garage-layout-diff" garage_peer_keygen "cryptic-net/cmd/garage-peer-keygen" - garage_update_global_bucket "cryptic-net/cmd/garage-update-global-bucket" nebula_entrypoint "cryptic-net/cmd/nebula-entrypoint" - nebula_update_global_bucket "cryptic-net/cmd/nebula-update-global-bucket" + update_global_bucket "cryptic-net/cmd/update-global-bucket" "fmt" "os" ) @@ -36,9 +35,8 @@ var mainFns = []mainFn{ {"garage-entrypoint", garage_entrypoint.Main}, {"garage-layout-diff", garage_layout_diff.Main}, {"garage-peer-keygen", garage_peer_keygen.Main}, - {"garage-update-global-bucket", garage_update_global_bucket.Main}, {"nebula-entrypoint", nebula_entrypoint.Main}, - {"nebula-update-global-bucket", nebula_update_global_bucket.Main}, + {"update-global-bucket", update_global_bucket.Main}, } var mainFnsMap = func() map[string]mainFn { diff --git a/go-workspace/src/cmd/entrypoint/daemon.go b/go-workspace/src/cmd/entrypoint/daemon.go index d6ab91d..7dc1122 100644 --- a/go-workspace/src/cmd/entrypoint/daemon.go +++ b/go-workspace/src/cmd/entrypoint/daemon.go @@ -7,14 +7,16 @@ import ( "fmt" "io" "io/ioutil" + "net" "os" "path/filepath" + "strconv" "sync" "time" crypticnet "cryptic-net" "cryptic-net/bootstrap" - bootstrap_creator "cryptic-net/bootstrap/creator" + "cryptic-net/garage" "cryptic-net/yamlutil" "github.com/cryptic-io/pmux/pmuxlib" @@ -108,21 +110,30 @@ func writeBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error { // creates a new bootstrap file using available information from the network. If // the new bootstrap file is different than the existing one, the existing one // is overwritten, ReloadBootstrap is called on env, true is returned. -func reloadBootstrap(env *crypticnet.Env) (bool, error) { +func reloadBootstrap(env *crypticnet.Env, s3Client garage.S3APIClient) (bool, error) { + + newHosts, err := bootstrap.GetGarageBootstrapHosts(env.Context, s3Client) + if err != nil { + return false, fmt.Errorf("getting hosts from garage: %w", err) + } + + newHostsHash, err := bootstrap.HostsHash(newHosts) + if err != nil { + return false, fmt.Errorf("calculating hash of new hosts: %w", err) + } + + currHostsHash, err := bootstrap.HostsHash(env.Bootstrap.Hosts) + if err != nil { + return false, fmt.Errorf("calculating hash of current hosts: %w", err) + } + + if bytes.Equal(newHostsHash, currHostsHash) { + return false, nil + } buf := new(bytes.Buffer) - - if err := bootstrap_creator.NewForThisHost(env, buf); err != nil { - return false, fmt.Errorf("generating new bootstrap from env: %w", err) - } - - newBootstrap, err := bootstrap.FromReader(bytes.NewReader(buf.Bytes())) - if err != nil { - return false, fmt.Errorf("parsing bootstrap which was just created: %w", err) - } - - if bytes.Equal(newBootstrap.Hash, env.Bootstrap.Hash) { - return false, nil + if err := env.Bootstrap.WithHosts(newHosts).WriteTo(buf); err != nil { + return false, fmt.Errorf("writing new bootstrap file: %w", err) } if err := writeBootstrapToDataDir(env, buf); err != nil { @@ -135,9 +146,10 @@ func reloadBootstrap(env *crypticnet.Env) (bool, error) { // runs a single pmux process ofor daemon, returning only once the env.Context // has been canceled or bootstrap info has been changed. This will always block // until the spawned pmux has returned. -func runDaemonPmuxOnce(env *crypticnet.Env) error { +func runDaemonPmuxOnce(env *crypticnet.Env, s3Client garage.S3APIClient) error { thisHost := env.Bootstrap.ThisHost() + thisDaemon := env.ThisDaemon() fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP) pmuxProcConfigs := []pmuxlib.ProcessConfig{ @@ -154,13 +166,39 @@ func runDaemonPmuxOnce(env *crypticnet.Env) error { Args: []string{ "wait-for-ip", thisHost.Nebula.IP, - "bash", "dnsmasq-entrypoint", }, }, } - if len(env.ThisDaemon().Storage.Allocations) > 0 { + { + var args []string + + if allocs := thisDaemon.Storage.Allocations; len(allocs) > 0 { + for _, alloc := range allocs { + args = append( + args, + "wait-for", + net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)), + "--", + ) + } + } else { + args = []string{ + "wait-for-ip", + thisHost.Nebula.IP, + } + } + + pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{ + Name: "update-global-bucket", + Cmd: "bash", + Args: append(args, "update-global-bucket"), + NoRestartOn: []int{0}, + }) + } + + if len(thisDaemon.Storage.Allocations) > 0 { pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{ Name: "garage", Cmd: "bash", @@ -204,7 +242,7 @@ func runDaemonPmuxOnce(env *crypticnet.Env) error { fmt.Fprintln(os.Stderr, "checking for changes to bootstrap") - if changed, err := reloadBootstrap(env); err != nil { + if changed, err := reloadBootstrap(env, s3Client); err != nil { return fmt.Errorf("reloading bootstrap: %w", err) } else if changed { @@ -243,6 +281,11 @@ var subCmdDaemon = subCmd{ env := subCmdCtx.env + s3Client, err := env.GlobalBucketS3APIClient() + if err != nil { + return fmt.Errorf("creating client for global bucket: %w", err) + } + appDirPath := env.AppDirPath builtinDaemonYmlPath := filepath.Join(appDirPath, "etc", "daemon.yml") @@ -326,7 +369,7 @@ var subCmdDaemon = subCmd{ for { - if err := runDaemonPmuxOnce(env); errors.Is(err, context.Canceled) { + if err := runDaemonPmuxOnce(env, s3Client); errors.Is(err, context.Canceled) { return nil } else if err != nil { diff --git a/go-workspace/src/cmd/entrypoint/hosts.go b/go-workspace/src/cmd/entrypoint/hosts.go index 4c80a16..6ae8cdb 100644 --- a/go-workspace/src/cmd/entrypoint/hosts.go +++ b/go-workspace/src/cmd/entrypoint/hosts.go @@ -1,29 +1,20 @@ package entrypoint import ( - "bytes" "cryptic-net/bootstrap" - bootstrap_creator "cryptic-net/bootstrap/creator" - "cryptic-net/garage" + "cryptic-net/nebula" "cryptic-net/tarutil" "errors" "fmt" "io/fs" "net" "os" - "path/filepath" "regexp" + "sort" - "github.com/minio/minio-go/v7" "gopkg.in/yaml.v3" ) -const nebulaHostPathPrefix = "nebula/hosts/" - -func nebulaHostPath(name string) string { - return filepath.Join(nebulaHostPathPrefix, name+".yml") -} - var hostNameRegexp = regexp.MustCompile(`^[a-z][a-z0-9\-]*$`) func validateHostName(name string) error { @@ -78,30 +69,14 @@ var subCmdHostsAdd = subCmd{ return fmt.Errorf("creating client for global bucket: %w", err) } - nebulaHost := bootstrap.NebulaHost{ + host := bootstrap.Host{ Name: *name, - IP: *ip, + Nebula: bootstrap.NebulaHost{ + IP: *ip, + }, } - bodyBuf := new(bytes.Buffer) - - if err := yaml.NewEncoder(bodyBuf).Encode(nebulaHost); err != nil { - return fmt.Errorf("marshaling nebula host to yaml: %w", err) - } - - filePath := nebulaHostPath(*name) - - _, err = client.PutObject( - env.Context, garage.GlobalBucket, filePath, - bodyBuf, int64(bodyBuf.Len()), - minio.PutObjectOptions{}, - ) - - if err != nil { - return fmt.Errorf("writing to %q in global bucket: %w", filePath, err) - } - - return nil + return bootstrap.PutGarageBoostrapHost(env.Context, client, host) }, } @@ -118,51 +93,19 @@ var subCmdHostsList = subCmd{ return fmt.Errorf("creating client for global bucket: %w", err) } - objInfoCh := client.ListObjects( - env.Context, garage.GlobalBucket, - minio.ListObjectsOptions{ - Prefix: nebulaHostPathPrefix, - }, - ) - - for { - select { - case <-env.Context.Done(): - return env.Context.Err() - - case objInfo, ok := <-objInfoCh: - - if !ok { - return nil - } else if objInfo.Err != nil { - return objInfo.Err - } - - obj, err := client.GetObject( - env.Context, garage.GlobalBucket, objInfo.Key, - minio.GetObjectOptions{}, - ) - - if err != nil { - return fmt.Errorf("retrieving object %q from global bucket: %w", objInfo.Key, err) - } - - var nebulaHost bootstrap.NebulaHost - - err = yaml.NewDecoder(obj).Decode(&nebulaHost) - obj.Close() - - if err != nil { - return fmt.Errorf("yaml decoding %q from global bucket: %w", objInfo.Key, err) - } - - fmt.Fprintf( - os.Stdout, "%s\t%s\n", - nebulaHost.Name, nebulaHost.IP, - ) - } + hostsMap, err := bootstrap.GetGarageBootstrapHosts(env.Context, client) + if err != nil { + return fmt.Errorf("retrieving hosts from garage: %w", err) } + hosts := make([]bootstrap.Host, 0, len(hostsMap)) + for _, host := range hostsMap { + hosts = append(hosts, host) + } + + sort.Slice(hosts, func(i, j int) bool { return hosts[i].Name < hosts[j].Name }) + + return yaml.NewEncoder(os.Stdout).Encode(hosts) }, } @@ -176,7 +119,7 @@ var subCmdHostsDelete = subCmd{ name := flags.StringP( "name", "n", "", - "Name of the new host", + "Name of the host to delete", ) if err := flags.Parse(subCmdCtx.args); err != nil { @@ -189,25 +132,12 @@ var subCmdHostsDelete = subCmd{ env := subCmdCtx.env - filePath := nebulaHostPath(*name) - client, err := env.GlobalBucketS3APIClient() if err != nil { return fmt.Errorf("creating client for global bucket: %w", err) } - err = client.RemoveObject( - env.Context, garage.GlobalBucket, filePath, - minio.RemoveObjectOptions{}, - ) - - if garage.IsKeyNotFound(err) { - return fmt.Errorf("host %q not found", *name) - } else if err != nil { - return fmt.Errorf("removing object %q from global bucket: %w", filePath, err) - } - - return nil + return bootstrap.RemoveGarageBootstrapHost(env.Context, client, *name) }, } @@ -258,12 +188,51 @@ var subCmdHostsMakeBootstrap = subCmd{ return errors.New("--name and --admin-path are required") } + env := subCmdCtx.env + adminFS, err := readAdminFS(*adminPath) if err != nil { return fmt.Errorf("reading admin.tgz with --admin-path of %q: %w", *adminPath, err) } - return bootstrap_creator.NewForHost(subCmdCtx.env, adminFS, *name, os.Stdout) + client, err := env.GlobalBucketS3APIClient() + if err != nil { + return fmt.Errorf("creating client for global bucket: %w", err) + } + + // NOTE this isn't _technically_ required, but if the `hosts add` + // command for this host has been run recently then it might not have + // made it into the bootstrap file yet, and so won't be in + // `env.Bootstrap`. + hosts, err := bootstrap.GetGarageBootstrapHosts(env.Context, client) + if err != nil { + return fmt.Errorf("retrieving host info from garage: %w", err) + } + + host, ok := hosts[*name] + if !ok { + return fmt.Errorf("couldn't find host into for %q in garage, has `cryptic-net hosts add` been run yet?", *name) + } + + nebulaHostCert, err := nebula.NewHostCert(adminFS, host) + if err != nil { + return fmt.Errorf("creating new nebula host key/cert: %w", err) + } + + newBootstrap := bootstrap.Bootstrap{ + Hosts: hosts, + HostName: *name, + + NebulaCertsCACert: nebulaHostCert.CACert, + NebulaCertsHostCert: nebulaHostCert.HostCert, + NebulaCertsHostKey: nebulaHostCert.HostKey, + + // TODO these should use adminFS + GarageRPCSecret: env.Bootstrap.GarageRPCSecret, + GarageGlobalBucketS3APICredentials: env.Bootstrap.GarageGlobalBucketS3APICredentials, + } + + return newBootstrap.WriteTo(os.Stdout) }, } diff --git a/go-workspace/src/cmd/garage-entrypoint/main.go b/go-workspace/src/cmd/garage-entrypoint/main.go index 995354d..87228a7 100644 --- a/go-workspace/src/cmd/garage-entrypoint/main.go +++ b/go-workspace/src/cmd/garage-entrypoint/main.go @@ -125,12 +125,5 @@ func Main() { NoRestartOn: []int{0}, }) - pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{ - Name: "garage-update-global-bucket", - Cmd: "bash", - Args: waitForArgs(env, "cryptic-net-main", "garage-update-global-bucket"), - NoRestartOn: []int{0}, - }) - pmuxlib.Run(env.Context, pmuxlib.Config{Processes: pmuxProcConfigs}) } diff --git a/go-workspace/src/cmd/garage-update-global-bucket/main.go b/go-workspace/src/cmd/garage-update-global-bucket/main.go deleted file mode 100644 index 85da6f2..0000000 --- a/go-workspace/src/cmd/garage-update-global-bucket/main.go +++ /dev/null @@ -1,90 +0,0 @@ -package garage_update_global_bucket - -import ( - "bytes" - crypticnet "cryptic-net" - "cryptic-net/bootstrap" - "cryptic-net/garage" - "fmt" - "log" - "path/filepath" - - "github.com/minio/minio-go/v7" - "gopkg.in/yaml.v3" -) - -func updateGlobalBucket(env *crypticnet.Env) error { - - ctx := env.Context - - client, err := env.GlobalBucketS3APIClient() - if err != nil { - return fmt.Errorf("creating client for global bucket: %w", err) - } - - filePath := filepath.Join( - "garage/hosts", - env.Bootstrap.HostName+".yml", - ) - - daemon := env.ThisDaemon() - - if len(daemon.Storage.Allocations) == 0 { - - err := client.RemoveObject( - ctx, garage.GlobalBucket, filePath, - minio.RemoveObjectOptions{}, - ) - - if garage.IsKeyNotFound(err) { - return nil - } else if err != nil { - return fmt.Errorf("removing %q from global bucket: %w", filePath, err) - } - - return nil - } - - var garageHost bootstrap.GarageHost - - for _, alloc := range daemon.Storage.Allocations { - - garageHostInstance := bootstrap.GarageHostInstance{ - RPCPort: alloc.RPCPort, - S3APIPort: alloc.S3APIPort, - WebPort: alloc.WebPort, - } - - garageHost.Instances = append(garageHost.Instances, garageHostInstance) - } - - buf := new(bytes.Buffer) - - if err := yaml.NewEncoder(buf).Encode(garageHost); err != nil { - return fmt.Errorf("yaml encoding garage host data: %w", err) - } - - _, err = client.PutObject( - ctx, garage.GlobalBucket, filePath, buf, int64(buf.Len()), - minio.PutObjectOptions{}, - ) - - if err != nil { - return fmt.Errorf("writing to %q in global bucket: %w", filePath, err) - } - - return nil -} - -func Main() { - - env, err := crypticnet.ReadEnv() - - if err != nil { - log.Fatalf("reading envvars: %v", err) - } - - if err := updateGlobalBucket(env); err != nil { - log.Fatalf("updating global bucket: %v", err) - } -} diff --git a/go-workspace/src/cmd/nebula-entrypoint/main.go b/go-workspace/src/cmd/nebula-entrypoint/main.go index bb434d5..ebc7a43 100644 --- a/go-workspace/src/cmd/nebula-entrypoint/main.go +++ b/go-workspace/src/cmd/nebula-entrypoint/main.go @@ -4,12 +4,12 @@ import ( "cryptic-net/yamlutil" "log" "net" + "os" "path/filepath" "strconv" + "syscall" crypticnet "cryptic-net" - - "github.com/cryptic-io/pmux/pmuxlib" ) func Main() { @@ -122,19 +122,13 @@ func Main() { log.Fatalf("writing nebula.yml to %q: %v", nebulaYmlPath, err) } - pmuxlib.Run(env.Context, pmuxlib.Config{Processes: []pmuxlib.ProcessConfig{ - { - Name: "nebula-update-global-bucket", - Cmd: "cryptic-net-main", - Args: []string{ - "nebula-update-global-bucket", - }, - NoRestartOn: []int{0}, - }, - { - Name: "nebula", - Cmd: "nebula", - Args: []string{"-config", nebulaYmlPath}, - }, - }}) + var ( + binPath = env.BinPath("nebula") + args = []string{"nebula", "-config", nebulaYmlPath} + cliEnv = os.Environ() + ) + + if err := syscall.Exec(binPath, args, cliEnv); err != nil { + log.Fatalf("calling exec(%q, %#v, %#v)", binPath, args, cliEnv) + } } diff --git a/go-workspace/src/cmd/nebula-update-global-bucket/main.go b/go-workspace/src/cmd/nebula-update-global-bucket/main.go deleted file mode 100644 index 555f2be..0000000 --- a/go-workspace/src/cmd/nebula-update-global-bucket/main.go +++ /dev/null @@ -1,62 +0,0 @@ -package nebula_update_global_bucket - -import ( - "bytes" - crypticnet "cryptic-net" - "cryptic-net/garage" - "fmt" - "log" - "path/filepath" - - "github.com/minio/minio-go/v7" - "gopkg.in/yaml.v3" -) - -func updateGlobalBucket(env *crypticnet.Env) error { - - ctx := env.Context - - client, err := env.GlobalBucketS3APIClient() - if err != nil { - return fmt.Errorf("creating client for global bucket: %w", err) - } - - daemon := env.ThisDaemon() - - host := env.Bootstrap.ThisHost() - - host.Nebula.Name = host.Name - host.Nebula.PublicAddr = daemon.VPN.PublicAddr - - buf := new(bytes.Buffer) - - if err := yaml.NewEncoder(buf).Encode(host.Nebula); err != nil { - return fmt.Errorf("yaml encoding garage host data: %w", err) - } - - filePath := filepath.Join("nebula/hosts", host.Name+".yml") - - _, err = client.PutObject( - ctx, garage.GlobalBucket, filePath, buf, int64(buf.Len()), - minio.PutObjectOptions{}, - ) - - if err != nil { - return fmt.Errorf("writing to %q in global bucket: %w", filePath, err) - } - - return nil -} - -func Main() { - - env, err := crypticnet.ReadEnv() - - if err != nil { - log.Fatalf("reading envvars: %v", err) - } - - if err := updateGlobalBucket(env); err != nil { - log.Fatalf("updating global bucket: %v", err) - } -} diff --git a/go-workspace/src/cmd/update-global-bucket/main.go b/go-workspace/src/cmd/update-global-bucket/main.go new file mode 100644 index 0000000..8729495 --- /dev/null +++ b/go-workspace/src/cmd/update-global-bucket/main.go @@ -0,0 +1,54 @@ +package update_global_bucket + +import ( + crypticnet "cryptic-net" + "cryptic-net/bootstrap" + "log" +) + +func Main() { + + env, err := crypticnet.ReadEnv() + if err != nil { + log.Fatalf("reading envvars: %v", err) + } + + client, err := env.GlobalBucketS3APIClient() + if err != nil { + log.Fatalf("creating client for global bucket: %v", err) + } + + host := env.Bootstrap.ThisHost() + + // We update the Host for this host in place, prior to writing it via the + // bootstrap method. We want to ensure that any changes made via daemon are + // reflected into the bootstrap data which is pushed up. + // + // TODO it'd be better if this was done within the daemon command itself, + // prior to any sub-processes being started. This would help us avoid this + // weird logic here, and would prevent all sub-processes from needing to be + // restarted the first time the daemon is started after daemon.yml is + // modified. + daemon := env.ThisDaemon() + + host.Nebula.PublicAddr = daemon.VPN.PublicAddr + + host.Garage = nil + + if allocs := daemon.Storage.Allocations; len(allocs) > 0 { + + host.Garage = new(bootstrap.GarageHost) + + for _, alloc := range allocs { + host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{ + RPCPort: alloc.RPCPort, + S3APIPort: alloc.S3APIPort, + WebPort: alloc.WebPort, + }) + } + } + + if err := bootstrap.PutGarageBoostrapHost(env.Context, client, host); err != nil { + log.Fatal(err) + } +} diff --git a/go-workspace/src/nebula/nebula.go b/go-workspace/src/nebula/nebula.go index 55cef5d..4623158 100644 --- a/go-workspace/src/nebula/nebula.go +++ b/go-workspace/src/nebula/nebula.go @@ -28,22 +28,22 @@ var ipCIDRMask = func() net.IPMask { // HostCert contains the certificate and private key files which will need to // be present on a particular host. Each file is PEM encoded. type HostCert struct { - CACert []byte - HostKey []byte - HostCert []byte + CACert string + HostKey string + HostCert string } // CACert contains the certificate and private files which can be used to create // HostCerts. Each file is PEM encoded. type CACert struct { - CACert []byte - CAKey []byte + CACert string + CAKey string } // NewHostCert generates a new key/cert for a nebula host using the CA key // which will be found in the adminFS. func NewHostCert( - adminFS fs.FS, host bootstrap.NebulaHost, + adminFS fs.FS, host bootstrap.Host, ) ( HostCert, error, ) { @@ -78,9 +78,9 @@ func NewHostCert( expireAt := caCrt.Details.NotAfter.Add(-1 * time.Second) - ip := net.ParseIP(host.IP) + ip := net.ParseIP(host.Nebula.IP) if ip == nil { - return HostCert{}, fmt.Errorf("invalid host ip %q", host.IP) + return HostCert{}, fmt.Errorf("invalid host ip %q", host.Nebula.IP) } ipNet := &net.IPNet{ @@ -126,9 +126,9 @@ func NewHostCert( } return HostCert{ - CACert: caCrtPEM, - HostKey: hostKeyPEM, - HostCert: hostCrtPEM, + CACert: string(caCrtPEM), + HostKey: string(hostKeyPEM), + HostCert: string(hostCrtPEM), }, nil } @@ -166,7 +166,7 @@ func NewCACert(domain string) (CACert, error) { } return CACert{ - CACert: caCrtPem, - CAKey: caKeyPEM, + CACert: string(caCrtPem), + CAKey: string(caKeyPEM), }, nil }