micropelago/isle
4
1
Fork 0
Code Issues 1 Pull Requests Projects Releases 5 Activity

More big refactoring leading up to network creation

Browse Source 
The `bootstrap/creator` package is gone, almost as quickly as it
arrived. The `Bootstrap` type is now able to write its own tgz file, and
the two places where bootstrap files are being created pull the data
down to do so and create the `Bootstrap` structs directly.

The structure of the bootstrap file itself has been changed, now there's
just a single `hosts` directory which contains files which are yaml
encodings of the `Host` type, rather than having it be split into
`nebula` and `garage` directories. This makes creating bootstrap files a
lot easier.
This commit is contained in:
Brian Picciano 2022-10-15 18:41:07 +02:00
parent 836e69735d
commit af7c8dde32
16 changed files with 474 additions and 573 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
go-workspace/src/bootstrap/bootstrap.go
View File

@ -5,16 +5,21 @@ import (
"cryptic-net/garage"
"cryptic-net/tarutil"
"cryptic-net/yamlutil"
"crypto/sha512"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"sort"
"strings"
"gopkg.in/yaml.v3"
)
// Paths within the bootstrap FS which for general data.
const (
HostNamePath = "hostname"
hostNamePath = "hostname"
)
// Bootstrap is used for accessing all information contained within a
@ -32,14 +37,6 @@ type Bootstrap struct {
GarageRPCSecret string
GarageGlobalBucketS3APICredentials garage.S3APICredentials
// Hash is a determinstic hash of the contents of the bootstrap file. This
// will be populated when parsing a Bootstrap from a bootstrap.tgz, but will
// be ignored when creating a new bootstrap.tgz.
Hash []byte
// DEPRECATED do not use
FS fs.FS
}
// FromFS loads a Boostrap instance from the given fs.FS, which presumably
@ -51,8 +48,6 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
err error
)
b.FS = bootstrapFS
if b.Hosts, err = loadHosts(bootstrapFS); err != nil {
return Bootstrap{}, fmt.Errorf("loading hosts info from fs: %w", err)
}
@ -60,7 +55,7 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
if err = yamlutil.LoadYamlFSFile(
&b.GarageGlobalBucketS3APICredentials,
bootstrapFS,
GarageGlobalBucketKeyYmlPath,
garageGlobalBucketKeyYmlPath,
); err != nil {
return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", b.GarageGlobalBucketS3APICredentials, err)
}
@ -69,11 +64,11 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
into *string
path string
}{
{&b.HostName, HostNamePath},
{&b.NebulaCertsCACert, NebulaCertsCACertPath},
{&b.NebulaCertsHostCert, NebulaCertsHostCertPath},
{&b.NebulaCertsHostKey, NebulaCertsHostKeyPath},
{&b.GarageRPCSecret, GarageRPCSecretPath},
{&b.HostName, hostNamePath},
{&b.NebulaCertsCACert, nebulaCertsCACertPath},
{&b.NebulaCertsHostCert, nebulaCertsHostCertPath},
{&b.NebulaCertsHostKey, nebulaCertsHostKeyPath},
{&b.GarageRPCSecret, garageRPCSecretPath},
}
for _, f := range filesToLoadAsString {
@ -87,10 +82,6 @@ func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
// TODO confirm if this is necessary
b.GarageRPCSecret = strings.TrimSpace(b.GarageRPCSecret)
if b.Hash, err = fs.ReadFile(bootstrapFS, tarutil.HashBinPath); err != nil {
return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", tarutil.HashBinPath, err)
}
return b, nil
}
@ -117,6 +108,48 @@ func FromFile(path string) (Bootstrap, error) {
return FromReader(f)
}
// WriteTo writes the Bootstrap as a new bootstrap.tgz to the given io.Writer.
func (b Bootstrap) WriteTo(into io.Writer) error {
w := tarutil.NewTGZWriter(into)
filesToWriteAsString := []struct {
value string
path string
}{
{b.HostName, hostNamePath},
{b.NebulaCertsCACert, nebulaCertsCACertPath},
{b.NebulaCertsHostCert, nebulaCertsHostCertPath},
{b.NebulaCertsHostKey, nebulaCertsHostKeyPath},
{b.GarageRPCSecret, garageRPCSecretPath},
}
for _, f := range filesToWriteAsString {
w.WriteFileBytes(f.path, []byte(f.value))
}
garageGlobalBucketKeyB, err := yaml.Marshal(b.GarageGlobalBucketS3APICredentials)
if err != nil {
return fmt.Errorf("yaml encoding garage global bucket creds: %w", err)
}
w.WriteFileBytes(garageGlobalBucketKeyYmlPath, garageGlobalBucketKeyB)
for _, host := range b.Hosts {
hostB, err := yaml.Marshal(host)
if err != nil {
return fmt.Errorf("yaml encoding host %#v: %w", host, err)
}
path := filepath.Join(hostsDirPath, host.Name+".yml")
w.WriteFileBytes(path, hostB)
}
return w.Close()
}
// ThisHost is a shortcut for b.Hosts[b.HostName], but will panic if the
// HostName isn't found in the Hosts map.
func (b Bootstrap) ThisHost() Host {
@ -128,3 +161,38 @@ func (b Bootstrap) ThisHost() Host {
return host
}
// Hash returns a deterministic hash of the given hosts map.
func HostsHash(hostsMap map[string]Host) ([]byte, error) {
hosts := make([]Host, 0, len(hostsMap))
for _, host := range hostsMap {
hosts = append(hosts, host)
}
sort.Slice(hosts, func(i, j int) bool { return hosts[i].Name < hosts[j].Name })
h := sha512.New()
if err := yaml.NewEncoder(h).Encode(hosts); err != nil {
return nil, err
}
return h.Sum(nil), nil
}
// WithHosts returns a copy of the Bootstrap with the given set of Hosts applied
// to it. It will _not_ overwrite the Host for _this_ host, however.
func (b Bootstrap) WithHosts(hosts map[string]Host) Bootstrap {
hostsCopy := make(map[string]Host, len(hosts))
for name, host := range hosts {
hostsCopy[name] = host
}
hostsCopy[b.HostName] = b.ThisHost()
b.Hosts = hostsCopy
return b
}

110
go-workspace/src/bootstrap/creator/creator.go
View File

@ -1,110 +0,0 @@
// Package creator is responsible for creating bootstrap files. It exists
// separately from the main bootstrap package in order to prevent import loops
// due to its use of crypticnet.Env.
package creator
import (
"context"
crypticnet "cryptic-net"
"cryptic-net/bootstrap"
"cryptic-net/nebula"
"cryptic-net/tarutil"
"fmt"
"io"
"io/fs"
)
func newBootstrap(
ctx context.Context,
into io.Writer,
hostname provider,
nebulaCertsCACert provider,
nebulaCertsHostCert provider,
nebulaCertsHostKey provider,
nebulaHosts provider,
garageRPCSecret provider,
garageGlobalBucketKey provider,
garageHosts provider,
) error {
pairs := []struct {
path string
provider provider
}{
{bootstrap.HostNamePath, hostname},
{bootstrap.NebulaCertsCACertPath, nebulaCertsCACert},
{bootstrap.NebulaCertsHostCertPath, nebulaCertsHostCert},
{bootstrap.NebulaCertsHostKeyPath, nebulaCertsHostKey},
{bootstrap.NebulaHostsDirPath, nebulaHosts},
{bootstrap.GarageRPCSecretPath, garageRPCSecret},
{bootstrap.GarageGlobalBucketKeyYmlPath, garageGlobalBucketKey},
{bootstrap.GarageHostsDirPath, garageHosts},
}
w := tarutil.NewTGZWriter(into)
for _, pair := range pairs {
if err := pair.provider(ctx, w, pair.path); err != nil {
return fmt.Errorf("populating %q in new bootstrap: %w", pair.path, err)
}
}
return w.Close()
}
// NewForThisHost generates a new bootstrap file for the current host, based on
// the existing environment as well as data in garage.
func NewForThisHost(env *crypticnet.Env, into io.Writer) error {
client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
return newBootstrap(
env.Context,
into,
provideFromFS(env.Bootstrap.FS), // hostname
provideFromFS(env.Bootstrap.FS), // nebulaCertsCACert
provideFromFS(env.Bootstrap.FS), // nebulaCertsHostCert
provideFromFS(env.Bootstrap.FS), // nebulaCertsHostKey
provideDirFromGarage(client), // nebulaHosts
provideFromFS(env.Bootstrap.FS), // garageRPCSecret
provideFromFS(env.Bootstrap.FS), // garageGlobalBucketKey
provideDirFromGarage(client), // garageHosts
)
}
// NewForHost generates a new bootstrap file for an arbitrary host, based on the
// given admin file's FS and data in garage.
func NewForHost(env *crypticnet.Env, adminFS fs.FS, name string, into io.Writer) error {
host, ok := env.Bootstrap.Hosts[name]
if !ok {
return fmt.Errorf("unknown host %q, make sure host entry has been created", name)
}
client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
nebulaHostCert, err := nebula.NewHostCert(adminFS, host.Nebula)
if err != nil {
return fmt.Errorf("creating new nebula host key/cert: %w", err)
}
return newBootstrap(
env.Context,
into,
provideFromBytes([]byte(name)), // hostname
provideFromBytes(nebulaHostCert.CACert), // nebulaCertsCACert
provideFromBytes(nebulaHostCert.HostCert), // nebulaCertsHostCert
provideFromBytes(nebulaHostCert.HostKey), // nebulaCertsHostKey
provideDirFromGarage(client), // nebulaHosts
provideFromFS(adminFS), // garageRPCSecret
provideFromFS(adminFS), // garageGlobalBucketKey
provideDirFromGarage(client), // garageHosts
)
}

121
go-workspace/src/bootstrap/creator/provider.go
View File

@ -1,121 +0,0 @@
package creator
import (
"context"
"cryptic-net/garage"
"cryptic-net/tarutil"
"fmt"
"io/fs"
"github.com/minio/minio-go/v7"
)
// provider is a function which will populate the given filePath into the given
// TGZWriter. The path may be a file or a directory.
type provider func(context.Context, *tarutil.TGZWriter, string) error
func provideFromBytes(body []byte) provider {
return func(
ctx context.Context,
w *tarutil.TGZWriter,
filePath string,
) error {
w.WriteFileBytes(filePath, body)
return nil
}
}
func provideFromFS(srcFS fs.FS) provider {
return func(
ctx context.Context,
w *tarutil.TGZWriter,
filePath string,
) error {
return w.CopyFileFromFS(filePath, srcFS)
}
}
func provideDirFromFS(srcFS fs.FS) provider {
return func(
ctx context.Context,
w *tarutil.TGZWriter,
dirPath string,
) error {
return fs.WalkDir(
srcFS, dirPath,
func(filePath string, dirEntry fs.DirEntry, err error) error {
if err != nil {
return err
} else if dirEntry.IsDir() {
return nil
} else if err := w.CopyFileFromFS(filePath, srcFS); err != nil {
return fmt.Errorf("copying file %q: %w", filePath, err)
}
return nil
},
)
}
}
// TODO it'd be great if we could wrap a minio.Client into an fs.FS. That would
// get rid of a weird dependency in this package, and clean up this code a ton.
func provideDirFromGarage(client *minio.Client) provider {
return func(
ctx context.Context,
w *tarutil.TGZWriter,
dirPath string,
) error {
objInfoCh := client.ListObjects(
ctx, garage.GlobalBucket,
minio.ListObjectsOptions{
Prefix: dirPath,
Recursive: true,
},
)
for objInfo := range objInfoCh {
if objInfo.Err != nil {
return fmt.Errorf("listing objects: %w", objInfo.Err)
}
obj, err := client.GetObject(
ctx, garage.GlobalBucket, objInfo.Key, minio.GetObjectOptions{},
)
if err != nil {
return fmt.Errorf(
"retrieving object %q from global bucket: %w",
objInfo.Key, err,
)
}
objStat, err := obj.Stat()
if err != nil {
obj.Close()
return fmt.Errorf(
"stating object %q from global bucket: %w",
objInfo.Key, err,
)
}
w.WriteFile(objInfo.Key, objStat.Size, obj)
obj.Close()
}
return nil
}
}

5
go-workspace/src/bootstrap/garage.go
View File

@ -6,9 +6,8 @@ import (
// Paths within the bootstrap FS related to garage.
const (
GarageGlobalBucketKeyYmlPath = "garage/global-bucket-key.yml"
GarageRPCSecretPath = "garage/rpc-secret.txt"
GarageHostsDirPath = "garage/hosts"
garageGlobalBucketKeyYmlPath = "garage/global-bucket-key.yml"
garageRPCSecretPath = "garage/rpc-secret.txt"
)
// GaragePeers returns a Peer for each known garage instance in the network.

111
go-workspace/src/bootstrap/garage_global_bucket.go Normal file
View File

@ -0,0 +1,111 @@
package bootstrap
import (
"bytes"
"context"
"cryptic-net/garage"
"fmt"
"log"
"path/filepath"
"github.com/minio/minio-go/v7"
"gopkg.in/yaml.v3"
)
// Paths within garage's global bucket
const (
garageGlobalBucketBootstrapHostsDirPath = "bootstrap/hosts"
)
// PutGarageBoostrapHost places the <hostname>.yml file for the given host into
// garage so that other hosts are able to see relevant configuration for it.
//
// The given client should be for the global bucket.
func PutGarageBoostrapHost(
ctx context.Context, client garage.S3APIClient, host Host,
) error {
buf := new(bytes.Buffer)
if err := yaml.NewEncoder(buf).Encode(host); err != nil {
log.Fatalf("yaml encoding host data: %v", err)
}
filePath := filepath.Join(garageGlobalBucketBootstrapHostsDirPath, host.Name+".yml")
_, err := client.PutObject(
ctx, garage.GlobalBucket, filePath, buf, int64(buf.Len()),
minio.PutObjectOptions{},
)
if err != nil {
return fmt.Errorf("writing to %q in global bucket: %w", filePath, err)
}
return nil
}
// RemoveGarageBootstrapHost removes the <hostname>.yml for the given host from
// garage.
//
// The given client should be for the global bucket.
func RemoveGarageBootstrapHost(
ctx context.Context, client garage.S3APIClient, hostName string,
) error {
filePath := filepath.Join(garageGlobalBucketBootstrapHostsDirPath, hostName+".yml")
return client.RemoveObject(
ctx, garage.GlobalBucket, filePath,
minio.RemoveObjectOptions{},
)
}
// GetGarageBootstrapHosts loads the <hostname>.yml file for all hosts stored in
// garage.
//
// The given client should be for the global bucket.
func GetGarageBootstrapHosts(
ctx context.Context, client garage.S3APIClient,
) (
map[string]Host, error,
) {
hosts := map[string]Host{}
objInfoCh := client.ListObjects(
ctx, garage.GlobalBucket,
minio.ListObjectsOptions{
Prefix: garageGlobalBucketBootstrapHostsDirPath,
Recursive: true,
},
)
for objInfo := range objInfoCh {
if objInfo.Err != nil {
return nil, fmt.Errorf("listing objects: %w", objInfo.Err)
}
obj, err := client.GetObject(
ctx, garage.GlobalBucket, objInfo.Key, minio.GetObjectOptions{},
)
if err != nil {
return nil, fmt.Errorf("retrieving object %q: %w", objInfo.Key, err)
}
var host Host
err = yaml.NewDecoder(obj).Decode(&host)
obj.Close()
if err != nil {
return nil, fmt.Errorf("yaml decoding object %q: %w", objInfo.Key, err)
}
hosts[host.Name] = host
}
return hosts, nil
}

77
go-workspace/src/bootstrap/hosts.go
View File

@ -10,21 +10,26 @@ import (
"gopkg.in/yaml.v3"
)
// NebulaHost describes the contents of a `./nebula/hosts/<hostname>.yml` file.
const (
hostsDirPath = "hosts"
)
// NebulaHost describes the nebula configuration of a Host which is relevant for
// other hosts to know.
type NebulaHost struct {
Name string `yaml:"name"`
IP string `yaml:"ip"`
PublicAddr string `yaml:"public_addr,omitempty"`
}
// GarageHostInstance describes a single garage instance running on a host.
// GarageHost describes a single garage instance in the GarageHost.
type GarageHostInstance struct {
RPCPort int `yaml:"rpc_port"`
S3APIPort int `yaml:"s3_api_port"`
WebPort int `yaml:"web_port"`
}
// GarageHost describes the contents of a `./garage/hosts/<hostname>.yml` file.
// GarageHost describes the garage configuration of a Host which is relevant for
// other hosts to know.
type GarageHost struct {
Instances []GarageHostInstance `yaml:"instances"`
}
@ -32,12 +37,12 @@ type GarageHost struct {
// Host consolidates all information about a single host from the bootstrap
// file.
type Host struct {
Name string
Nebula NebulaHost
Garage *GarageHost
Name string `yaml:"name"`
Nebula NebulaHost `yaml:"nebula"`
Garage *GarageHost `yaml:"garage,omitempty"`
}
func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
func loadHostsLegacy(bootstrapFS fs.FS) (map[string]Host, error) {
hosts := map[string]Host{}
@ -51,7 +56,7 @@ func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
}
{
globPath := filepath.Join(NebulaHostsDirPath, "*.yml")
globPath := "nebula/hosts/*.yml"
nebulaHostFiles, err := fs.Glob(bootstrapFS, globPath)
if err != nil {
@ -77,7 +82,7 @@ func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
for hostName, host := range hosts {
garageHostPath := filepath.Join(GarageHostsDirPath, hostName+".yml")
garageHostPath := filepath.Join("garage/hosts", hostName+".yml")
var garageHost GarageHost
if err := readAsYaml(&garageHost, garageHostPath); errors.Is(err, fs.ErrNotExist) {
@ -92,3 +97,55 @@ func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
return hosts, nil
}
func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
hosts := map[string]Host{}
readAsYaml := func(into interface{}, path string) error {
b, err := fs.ReadFile(bootstrapFS, path)
if err != nil {
return fmt.Errorf("reading file from fs: %w", err)
}
return yaml.Unmarshal(b, into)
}
globPath := filepath.Join(hostsDirPath, "*.yml")
hostPaths, err := fs.Glob(bootstrapFS, globPath)
if err != nil {
return nil, fmt.Errorf("listing host files at %q in fs: %w", globPath, err)
}
for _, hostPath := range hostPaths {
hostName := filepath.Base(hostPath)
hostName = strings.TrimSuffix(hostName, filepath.Ext(hostName))
var host Host
if err := readAsYaml(&host, hostPath); err != nil {
return nil, fmt.Errorf("reading %q as yaml: %w", hostPath, err)
}
hosts[hostName] = host
}
if len(hosts) > 0 {
return hosts, nil
}
// We used to have the bootstrap file laid out differently. If no hosts were
// found then the bootstrap file is probably in that format.
hosts, err = loadHostsLegacy(bootstrapFS)
if err != nil {
return nil, fmt.Errorf("loading hosts in legacy layout from fs: %w", err)
}
if len(hosts) == 0 {
return nil, fmt.Errorf("failed to load any hosts from fs")
}
return hosts, nil
}

8
go-workspace/src/bootstrap/nebula.go
View File

@ -2,9 +2,7 @@ package bootstrap
// Paths within the bootstrap FS related to nebula.
const (
NebulaHostsDirPath = "nebula/hosts"
NebulaCertsCACertPath = "nebula/certs/ca.crt"
NebulaCertsHostCertPath = "nebula/certs/host.crt"
NebulaCertsHostKeyPath = "nebula/certs/host.key"
nebulaCertsCACertPath = "nebula/certs/ca.crt"
nebulaCertsHostCertPath = "nebula/certs/host.crt"
nebulaCertsHostKeyPath = "nebula/certs/host.key"
)

6
go-workspace/src/cmd/cryptic-net-main/main.go
View File

@ -19,9 +19,8 @@ import (
garage_entrypoint "cryptic-net/cmd/garage-entrypoint"
garage_layout_diff "cryptic-net/cmd/garage-layout-diff"
garage_peer_keygen "cryptic-net/cmd/garage-peer-keygen"
garage_update_global_bucket "cryptic-net/cmd/garage-update-global-bucket"
nebula_entrypoint "cryptic-net/cmd/nebula-entrypoint"
nebula_update_global_bucket "cryptic-net/cmd/nebula-update-global-bucket"
update_global_bucket "cryptic-net/cmd/update-global-bucket"
"fmt"
"os"
)
@ -36,9 +35,8 @@ var mainFns = []mainFn{
{"garage-entrypoint", garage_entrypoint.Main},
{"garage-layout-diff", garage_layout_diff.Main},
{"garage-peer-keygen", garage_peer_keygen.Main},
{"garage-update-global-bucket", garage_update_global_bucket.Main},
{"nebula-entrypoint", nebula_entrypoint.Main},
{"nebula-update-global-bucket", nebula_update_global_bucket.Main},
{"update-global-bucket", update_global_bucket.Main},
}
var mainFnsMap = func() map[string]mainFn {

81
go-workspace/src/cmd/entrypoint/daemon.go
View File

@ -7,14 +7,16 @@ import (
"fmt"
"io"
"io/ioutil"
"net"
"os"
"path/filepath"
"strconv"
"sync"
"time"
crypticnet "cryptic-net"
"cryptic-net/bootstrap"
bootstrap_creator "cryptic-net/bootstrap/creator"
"cryptic-net/garage"
"cryptic-net/yamlutil"
"github.com/cryptic-io/pmux/pmuxlib"
@ -108,21 +110,30 @@ func writeBootstrapToDataDir(env *crypticnet.Env, r io.Reader) error {
// creates a new bootstrap file using available information from the network. If
// the new bootstrap file is different than the existing one, the existing one
// is overwritten, ReloadBootstrap is called on env, true is returned.
func reloadBootstrap(env *crypticnet.Env) (bool, error) {
func reloadBootstrap(env *crypticnet.Env, s3Client garage.S3APIClient) (bool, error) {
newHosts, err := bootstrap.GetGarageBootstrapHosts(env.Context, s3Client)
if err != nil {
return false, fmt.Errorf("getting hosts from garage: %w", err)
}
newHostsHash, err := bootstrap.HostsHash(newHosts)
if err != nil {
return false, fmt.Errorf("calculating hash of new hosts: %w", err)
}
currHostsHash, err := bootstrap.HostsHash(env.Bootstrap.Hosts)
if err != nil {
return false, fmt.Errorf("calculating hash of current hosts: %w", err)
}
if bytes.Equal(newHostsHash, currHostsHash) {
return false, nil
}
buf := new(bytes.Buffer)
if err := bootstrap_creator.NewForThisHost(env, buf); err != nil {
return false, fmt.Errorf("generating new bootstrap from env: %w", err)
}
newBootstrap, err := bootstrap.FromReader(bytes.NewReader(buf.Bytes()))
if err != nil {
return false, fmt.Errorf("parsing bootstrap which was just created: %w", err)
}
if bytes.Equal(newBootstrap.Hash, env.Bootstrap.Hash) {
return false, nil
if err := env.Bootstrap.WithHosts(newHosts).WriteTo(buf); err != nil {
return false, fmt.Errorf("writing new bootstrap file: %w", err)
}
if err := writeBootstrapToDataDir(env, buf); err != nil {
@ -135,9 +146,10 @@ func reloadBootstrap(env *crypticnet.Env) (bool, error) {
// runs a single pmux process ofor daemon, returning only once the env.Context
// has been canceled or bootstrap info has been changed. This will always block
// until the spawned pmux has returned.
func runDaemonPmuxOnce(env *crypticnet.Env) error {
func runDaemonPmuxOnce(env *crypticnet.Env, s3Client garage.S3APIClient) error {
thisHost := env.Bootstrap.ThisHost()
thisDaemon := env.ThisDaemon()
fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP)
pmuxProcConfigs := []pmuxlib.ProcessConfig{
@ -154,13 +166,39 @@ func runDaemonPmuxOnce(env *crypticnet.Env) error {
Args: []string{
"wait-for-ip",
thisHost.Nebula.IP,
"bash",
"dnsmasq-entrypoint",
},
},
}
if len(env.ThisDaemon().Storage.Allocations) > 0 {
{
var args []string
if allocs := thisDaemon.Storage.Allocations; len(allocs) > 0 {
for _, alloc := range allocs {
args = append(
args,
"wait-for",
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
"--",
)
}
} else {
args = []string{
"wait-for-ip",
thisHost.Nebula.IP,
}
}
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
Name: "update-global-bucket",
Cmd: "bash",
Args: append(args, "update-global-bucket"),
NoRestartOn: []int{0},
})
}
if len(thisDaemon.Storage.Allocations) > 0 {
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
Name: "garage",
Cmd: "bash",
@ -204,7 +242,7 @@ func runDaemonPmuxOnce(env *crypticnet.Env) error {
fmt.Fprintln(os.Stderr, "checking for changes to bootstrap")
if changed, err := reloadBootstrap(env); err != nil {
if changed, err := reloadBootstrap(env, s3Client); err != nil {
return fmt.Errorf("reloading bootstrap: %w", err)
} else if changed {
@ -243,6 +281,11 @@ var subCmdDaemon = subCmd{
env := subCmdCtx.env
s3Client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
appDirPath := env.AppDirPath
builtinDaemonYmlPath := filepath.Join(appDirPath, "etc", "daemon.yml")
@ -326,7 +369,7 @@ var subCmdDaemon = subCmd{
for {
if err := runDaemonPmuxOnce(env); errors.Is(err, context.Canceled) {
if err := runDaemonPmuxOnce(env, s3Client); errors.Is(err, context.Canceled) {
return nil
} else if err != nil {

151
go-workspace/src/cmd/entrypoint/hosts.go
View File

@ -1,29 +1,20 @@
package entrypoint
import (
"bytes"
"cryptic-net/bootstrap"
bootstrap_creator "cryptic-net/bootstrap/creator"
"cryptic-net/garage"
"cryptic-net/nebula"
"cryptic-net/tarutil"
"errors"
"fmt"
"io/fs"
"net"
"os"
"path/filepath"
"regexp"
"sort"
"github.com/minio/minio-go/v7"
"gopkg.in/yaml.v3"
)
const nebulaHostPathPrefix = "nebula/hosts/"
func nebulaHostPath(name string) string {
return filepath.Join(nebulaHostPathPrefix, name+".yml")
}
var hostNameRegexp = regexp.MustCompile(`^[a-z][a-z0-9\-]*$`)
func validateHostName(name string) error {
@ -78,30 +69,14 @@ var subCmdHostsAdd = subCmd{
return fmt.Errorf("creating client for global bucket: %w", err)
}
nebulaHost := bootstrap.NebulaHost{
host := bootstrap.Host{
Name: *name,
IP: *ip,
Nebula: bootstrap.NebulaHost{
IP: *ip,
},
}
bodyBuf := new(bytes.Buffer)
if err := yaml.NewEncoder(bodyBuf).Encode(nebulaHost); err != nil {
return fmt.Errorf("marshaling nebula host to yaml: %w", err)
}
filePath := nebulaHostPath(*name)
_, err = client.PutObject(
env.Context, garage.GlobalBucket, filePath,
bodyBuf, int64(bodyBuf.Len()),
minio.PutObjectOptions{},
)
if err != nil {
return fmt.Errorf("writing to %q in global bucket: %w", filePath, err)
}
return nil
return bootstrap.PutGarageBoostrapHost(env.Context, client, host)
},
}
@ -118,51 +93,19 @@ var subCmdHostsList = subCmd{
return fmt.Errorf("creating client for global bucket: %w", err)
}
objInfoCh := client.ListObjects(
env.Context, garage.GlobalBucket,
minio.ListObjectsOptions{
Prefix: nebulaHostPathPrefix,
},
)
for {
select {
case <-env.Context.Done():
return env.Context.Err()
case objInfo, ok := <-objInfoCh:
if !ok {
return nil
} else if objInfo.Err != nil {
return objInfo.Err
}
obj, err := client.GetObject(
env.Context, garage.GlobalBucket, objInfo.Key,
minio.GetObjectOptions{},
)
if err != nil {
return fmt.Errorf("retrieving object %q from global bucket: %w", objInfo.Key, err)
}
var nebulaHost bootstrap.NebulaHost
err = yaml.NewDecoder(obj).Decode(&nebulaHost)
obj.Close()
if err != nil {
return fmt.Errorf("yaml decoding %q from global bucket: %w", objInfo.Key, err)
}
fmt.Fprintf(
os.Stdout, "%s\t%s\n",
nebulaHost.Name, nebulaHost.IP,
)
}
hostsMap, err := bootstrap.GetGarageBootstrapHosts(env.Context, client)
if err != nil {
return fmt.Errorf("retrieving hosts from garage: %w", err)
}
hosts := make([]bootstrap.Host, 0, len(hostsMap))
for _, host := range hostsMap {
hosts = append(hosts, host)
}
sort.Slice(hosts, func(i, j int) bool { return hosts[i].Name < hosts[j].Name })
return yaml.NewEncoder(os.Stdout).Encode(hosts)
},
}
@ -176,7 +119,7 @@ var subCmdHostsDelete = subCmd{
name := flags.StringP(
"name", "n", "",
"Name of the new host",
"Name of the host to delete",
)
if err := flags.Parse(subCmdCtx.args); err != nil {
@ -189,25 +132,12 @@ var subCmdHostsDelete = subCmd{
env := subCmdCtx.env
filePath := nebulaHostPath(*name)
client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
err = client.RemoveObject(
env.Context, garage.GlobalBucket, filePath,
minio.RemoveObjectOptions{},
)
if garage.IsKeyNotFound(err) {
return fmt.Errorf("host %q not found", *name)
} else if err != nil {
return fmt.Errorf("removing object %q from global bucket: %w", filePath, err)
}
return nil
return bootstrap.RemoveGarageBootstrapHost(env.Context, client, *name)
},
}
@ -258,12 +188,51 @@ var subCmdHostsMakeBootstrap = subCmd{
return errors.New("--name and --admin-path are required")
}
env := subCmdCtx.env
adminFS, err := readAdminFS(*adminPath)
if err != nil {
return fmt.Errorf("reading admin.tgz with --admin-path of %q: %w", *adminPath, err)
}
return bootstrap_creator.NewForHost(subCmdCtx.env, adminFS, *name, os.Stdout)
client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
// NOTE this isn't _technically_ required, but if the `hosts add`
// command for this host has been run recently then it might not have
// made it into the bootstrap file yet, and so won't be in
// `env.Bootstrap`.
hosts, err := bootstrap.GetGarageBootstrapHosts(env.Context, client)
if err != nil {
return fmt.Errorf("retrieving host info from garage: %w", err)
}
host, ok := hosts[*name]
if !ok {
return fmt.Errorf("couldn't find host into for %q in garage, has `cryptic-net hosts add` been run yet?", *name)
}
nebulaHostCert, err := nebula.NewHostCert(adminFS, host)
if err != nil {
return fmt.Errorf("creating new nebula host key/cert: %w", err)
}
newBootstrap := bootstrap.Bootstrap{
Hosts: hosts,
HostName: *name,
NebulaCertsCACert: nebulaHostCert.CACert,
NebulaCertsHostCert: nebulaHostCert.HostCert,
NebulaCertsHostKey: nebulaHostCert.HostKey,
// TODO these should use adminFS
GarageRPCSecret: env.Bootstrap.GarageRPCSecret,
GarageGlobalBucketS3APICredentials: env.Bootstrap.GarageGlobalBucketS3APICredentials,
}
return newBootstrap.WriteTo(os.Stdout)
},
}

7
go-workspace/src/cmd/garage-entrypoint/main.go
View File

@ -125,12 +125,5 @@ func Main() {
NoRestartOn: []int{0},
})
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
Name: "garage-update-global-bucket",
Cmd: "bash",
Args: waitForArgs(env, "cryptic-net-main", "garage-update-global-bucket"),
NoRestartOn: []int{0},
})
pmuxlib.Run(env.Context, pmuxlib.Config{Processes: pmuxProcConfigs})
}

90
go-workspace/src/cmd/garage-update-global-bucket/main.go
View File

@ -1,90 +0,0 @@
package garage_update_global_bucket
import (
"bytes"
crypticnet "cryptic-net"
"cryptic-net/bootstrap"
"cryptic-net/garage"
"fmt"
"log"
"path/filepath"
"github.com/minio/minio-go/v7"
"gopkg.in/yaml.v3"
)
func updateGlobalBucket(env *crypticnet.Env) error {
ctx := env.Context
client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
filePath := filepath.Join(
"garage/hosts",
env.Bootstrap.HostName+".yml",
)
daemon := env.ThisDaemon()
if len(daemon.Storage.Allocations) == 0 {
err := client.RemoveObject(
ctx, garage.GlobalBucket, filePath,
minio.RemoveObjectOptions{},
)
if garage.IsKeyNotFound(err) {
return nil
} else if err != nil {
return fmt.Errorf("removing %q from global bucket: %w", filePath, err)
}
return nil
}
var garageHost bootstrap.GarageHost
for _, alloc := range daemon.Storage.Allocations {
garageHostInstance := bootstrap.GarageHostInstance{
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
WebPort: alloc.WebPort,
}
garageHost.Instances = append(garageHost.Instances, garageHostInstance)
}
buf := new(bytes.Buffer)
if err := yaml.NewEncoder(buf).Encode(garageHost); err != nil {
return fmt.Errorf("yaml encoding garage host data: %w", err)
}
_, err = client.PutObject(
ctx, garage.GlobalBucket, filePath, buf, int64(buf.Len()),
minio.PutObjectOptions{},
)
if err != nil {
return fmt.Errorf("writing to %q in global bucket: %w", filePath, err)
}
return nil
}
func Main() {
env, err := crypticnet.ReadEnv()
if err != nil {
log.Fatalf("reading envvars: %v", err)
}
if err := updateGlobalBucket(env); err != nil {
log.Fatalf("updating global bucket: %v", err)
}
}

28
go-workspace/src/cmd/nebula-entrypoint/main.go
View File

@ -4,12 +4,12 @@ import (
"cryptic-net/yamlutil"
"log"
"net"
"os"
"path/filepath"
"strconv"
"syscall"
crypticnet "cryptic-net"
"github.com/cryptic-io/pmux/pmuxlib"
)
func Main() {
@ -122,19 +122,13 @@ func Main() {
log.Fatalf("writing nebula.yml to %q: %v", nebulaYmlPath, err)
}
pmuxlib.Run(env.Context, pmuxlib.Config{Processes: []pmuxlib.ProcessConfig{
{
Name: "nebula-update-global-bucket",
Cmd: "cryptic-net-main",
Args: []string{
"nebula-update-global-bucket",
},
NoRestartOn: []int{0},
},
{
Name: "nebula",
Cmd: "nebula",
Args: []string{"-config", nebulaYmlPath},
},
}})
var (
binPath = env.BinPath("nebula")
args = []string{"nebula", "-config", nebulaYmlPath}
cliEnv = os.Environ()
)
if err := syscall.Exec(binPath, args, cliEnv); err != nil {
log.Fatalf("calling exec(%q, %#v, %#v)", binPath, args, cliEnv)
}
}

62
go-workspace/src/cmd/nebula-update-global-bucket/main.go
View File

@ -1,62 +0,0 @@
package nebula_update_global_bucket
import (
"bytes"
crypticnet "cryptic-net"
"cryptic-net/garage"
"fmt"
"log"
"path/filepath"
"github.com/minio/minio-go/v7"
"gopkg.in/yaml.v3"
)
func updateGlobalBucket(env *crypticnet.Env) error {
ctx := env.Context
client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
daemon := env.ThisDaemon()
host := env.Bootstrap.ThisHost()
host.Nebula.Name = host.Name
host.Nebula.PublicAddr = daemon.VPN.PublicAddr
buf := new(bytes.Buffer)
if err := yaml.NewEncoder(buf).Encode(host.Nebula); err != nil {
return fmt.Errorf("yaml encoding garage host data: %w", err)
}
filePath := filepath.Join("nebula/hosts", host.Name+".yml")
_, err = client.PutObject(
ctx, garage.GlobalBucket, filePath, buf, int64(buf.Len()),
minio.PutObjectOptions{},
)
if err != nil {
return fmt.Errorf("writing to %q in global bucket: %w", filePath, err)
}
return nil
}
func Main() {
env, err := crypticnet.ReadEnv()
if err != nil {
log.Fatalf("reading envvars: %v", err)
}
if err := updateGlobalBucket(env); err != nil {
log.Fatalf("updating global bucket: %v", err)
}
}

54
go-workspace/src/cmd/update-global-bucket/main.go Normal file
View File

@ -0,0 +1,54 @@
package update_global_bucket
import (
crypticnet "cryptic-net"
"cryptic-net/bootstrap"
"log"
)
func Main() {
env, err := crypticnet.ReadEnv()
if err != nil {
log.Fatalf("reading envvars: %v", err)
}
client, err := env.GlobalBucketS3APIClient()
if err != nil {
log.Fatalf("creating client for global bucket: %v", err)
}
host := env.Bootstrap.ThisHost()
// We update the Host for this host in place, prior to writing it via the
// bootstrap method. We want to ensure that any changes made via daemon are
// reflected into the bootstrap data which is pushed up.
//
// TODO it'd be better if this was done within the daemon command itself,
// prior to any sub-processes being started. This would help us avoid this
// weird logic here, and would prevent all sub-processes from needing to be
// restarted the first time the daemon is started after daemon.yml is
// modified.
daemon := env.ThisDaemon()
host.Nebula.PublicAddr = daemon.VPN.PublicAddr
host.Garage = nil
if allocs := daemon.Storage.Allocations; len(allocs) > 0 {
host.Garage = new(bootstrap.GarageHost)
for _, alloc := range allocs {
host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
WebPort: alloc.WebPort,
})
}
}
if err := bootstrap.PutGarageBoostrapHost(env.Context, client, host); err != nil {
log.Fatal(err)
}
}

26
go-workspace/src/nebula/nebula.go
View File

@ -28,22 +28,22 @@ var ipCIDRMask = func() net.IPMask {
// HostCert contains the certificate and private key files which will need to
// be present on a particular host. Each file is PEM encoded.
type HostCert struct {
CACert []byte
HostKey []byte
HostCert []byte
CACert string
HostKey string
HostCert string
}
// CACert contains the certificate and private files which can be used to create
// HostCerts. Each file is PEM encoded.
type CACert struct {
CACert []byte
CAKey []byte
CACert string
CAKey string
}
// NewHostCert generates a new key/cert for a nebula host using the CA key
// which will be found in the adminFS.
func NewHostCert(
adminFS fs.FS, host bootstrap.NebulaHost,
adminFS fs.FS, host bootstrap.Host,
) (
HostCert, error,
) {
@ -78,9 +78,9 @@ func NewHostCert(
expireAt := caCrt.Details.NotAfter.Add(-1 * time.Second)
ip := net.ParseIP(host.IP)
ip := net.ParseIP(host.Nebula.IP)
if ip == nil {
return HostCert{}, fmt.Errorf("invalid host ip %q", host.IP)
return HostCert{}, fmt.Errorf("invalid host ip %q", host.Nebula.IP)
}
ipNet := &net.IPNet{
@ -126,9 +126,9 @@ func NewHostCert(
}
return HostCert{
CACert: caCrtPEM,
HostKey: hostKeyPEM,
HostCert: hostCrtPEM,
CACert: string(caCrtPEM),
HostKey: string(hostKeyPEM),
HostCert: string(hostCrtPEM),
}, nil
}
@ -166,7 +166,7 @@ func NewCACert(domain string) (CACert, error) {
}
return CACert{
CACert: caCrtPem,
CAKey: caKeyPEM,
CACert: string(caCrtPem),
CAKey: string(caKeyPEM),
}, nil
}