micropelago/isle
4
1
Fork 0
Code Issues 1 Pull Requests Projects Releases 5 Activity

Compare commits

base: micropelago:836e69735d71e96043d1ca46b45daf72e9503c47
Branches Tags
micropelago:main
micropelago:v0.0.2
micropelago:v0.0.1
micropelago:v0.0.1-rc2
micropelago:v0.0.1-rc1
micropelago:v0.0.1-rc0
..
compare: micropelago:24b7fe63394d6f2183c62b3c12acba4faf36a181
Branches Tags
micropelago:main
micropelago:v0.0.2
micropelago:v0.0.1
micropelago:v0.0.1-rc2
micropelago:v0.0.1-rc1
micropelago:v0.0.1-rc0

No commits in common. "836e69735d71e96043d1ca46b45daf72e9503c47" and "24b7fe63394d6f2183c62b3c12acba4faf36a181" have entirely different histories.
836e69735d ... 24b7fe6339

28 changed files with 671 additions and 815 deletions
Show Stats Expand all files Collapse all files

118
go-workspace/src/bootstrap/bootstrap.go
View File

@ -2,129 +2,33 @@
package bootstrap package bootstrap
import ( import (
"cryptic-net/garage"
"cryptic-net/tarutil" "cryptic-net/tarutil"
"cryptic-net/yamlutil"
"fmt" "fmt"
"io" "io"
"io/fs" "io/fs"
"os"
"strings"
) )
// Paths within the bootstrap FS which for general data. // GetHashFromFS returns the hash of the contents of the given bootstrap file.
const ( // It may return nil if the bootstrap file doesn't have a hash.
HostNamePath = "hostname" func GetHashFromFS(bootstrapFS fs.FS) ([]byte, error) {
)
// Bootstrap is used for accessing all information contained within a b, err := fs.ReadFile(bootstrapFS, tarutil.HashBinPath)
// bootstrap.tgz file.
//
// An instance of Bootstrap is read-only, the creator sub-package should be used
// to create new instances.
type Bootstrap struct {
Hosts map[string]Host
HostName string
NebulaCertsCACert string
NebulaCertsHostCert string
NebulaCertsHostKey string
GarageRPCSecret string
GarageGlobalBucketS3APICredentials garage.S3APICredentials
// Hash is a determinstic hash of the contents of the bootstrap file. This
// will be populated when parsing a Bootstrap from a bootstrap.tgz, but will
// be ignored when creating a new bootstrap.tgz.
Hash []byte
// DEPRECATED do not use
FS fs.FS
}
// FromFS loads a Boostrap instance from the given fs.FS, which presumably
// represents the file structure of a bootstrap.tgz file.
func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
var (
b Bootstrap
err error
)
b.FS = bootstrapFS
if b.Hosts, err = loadHosts(bootstrapFS); err != nil {
return Bootstrap{}, fmt.Errorf("loading hosts info from fs: %w", err)
}
if err = yamlutil.LoadYamlFSFile(
&b.GarageGlobalBucketS3APICredentials,
bootstrapFS,
GarageGlobalBucketKeyYmlPath,
); err != nil {
return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", b.GarageGlobalBucketS3APICredentials, err)
}
filesToLoadAsString := []struct {
into *string
path string
}{
{&b.HostName, HostNamePath},
{&b.NebulaCertsCACert, NebulaCertsCACertPath},
{&b.NebulaCertsHostCert, NebulaCertsHostCertPath},
{&b.NebulaCertsHostKey, NebulaCertsHostKeyPath},
{&b.GarageRPCSecret, GarageRPCSecretPath},
}
for _, f := range filesToLoadAsString {
body, err := fs.ReadFile(bootstrapFS, f.path)
if err != nil { if err != nil {
return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", f.path, err) return nil, fmt.Errorf("reading file %q from bootstrap fs: %w", tarutil.HashBinPath, err)
}
*f.into = string(body)
}
// TODO confirm if this is necessary
b.GarageRPCSecret = strings.TrimSpace(b.GarageRPCSecret)
if b.Hash, err = fs.ReadFile(bootstrapFS, tarutil.HashBinPath); err != nil {
return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", tarutil.HashBinPath, err)
} }
return b, nil return b, nil
} }
// FromReader reads a bootstrap.tgz file from the given io.Reader. // GetHashFromReader reads the given tgz file as an fs.FS, and passes that to
func FromReader(r io.Reader) (Bootstrap, error) { // GetHashFromFS.
func GetHashFromReader(r io.Reader) ([]byte, error) {
fs, err := tarutil.FSFromReader(r) bootstrapFS, err := tarutil.FSFromReader(r)
if err != nil { if err != nil {
return Bootstrap{}, fmt.Errorf("reading bootstrap.tgz: %w", err) return nil, fmt.Errorf("reading tar fs from reader: %w", err)
} }
return FromFS(fs) return GetHashFromFS(bootstrapFS)
}
// FromFile reads a bootstrap.tgz from a file at the given path.
func FromFile(path string) (Bootstrap, error) {
f, err := os.Open(path)
if err != nil {
return Bootstrap{}, fmt.Errorf("opening file: %w", err)
}
defer f.Close()
return FromReader(f)
}
// ThisHost is a shortcut for b.Hosts[b.HostName], but will panic if the
// HostName isn't found in the Hosts map.
func (b Bootstrap) ThisHost() Host {
host, ok := b.Hosts[b.HostName]
if !ok {
panic(fmt.Sprintf("hostname %q not defined in bootstrap's hosts", b.HostName))
}
return host
} }

110
go-workspace/src/bootstrap/creator/creator.go
View File

@ -1,110 +0,0 @@
// Package creator is responsible for creating bootstrap files. It exists
// separately from the main bootstrap package in order to prevent import loops
// due to its use of crypticnet.Env.
package creator
import (
"context"
crypticnet "cryptic-net"
"cryptic-net/bootstrap"
"cryptic-net/nebula"
"cryptic-net/tarutil"
"fmt"
"io"
"io/fs"
)
func newBootstrap(
ctx context.Context,
into io.Writer,
hostname provider,
nebulaCertsCACert provider,
nebulaCertsHostCert provider,
nebulaCertsHostKey provider,
nebulaHosts provider,
garageRPCSecret provider,
garageGlobalBucketKey provider,
garageHosts provider,
) error {
pairs := []struct {
path string
provider provider
}{
{bootstrap.HostNamePath, hostname},
{bootstrap.NebulaCertsCACertPath, nebulaCertsCACert},
{bootstrap.NebulaCertsHostCertPath, nebulaCertsHostCert},
{bootstrap.NebulaCertsHostKeyPath, nebulaCertsHostKey},
{bootstrap.NebulaHostsDirPath, nebulaHosts},
{bootstrap.GarageRPCSecretPath, garageRPCSecret},
{bootstrap.GarageGlobalBucketKeyYmlPath, garageGlobalBucketKey},
{bootstrap.GarageHostsDirPath, garageHosts},
}
w := tarutil.NewTGZWriter(into)
for _, pair := range pairs {
if err := pair.provider(ctx, w, pair.path); err != nil {
return fmt.Errorf("populating %q in new bootstrap: %w", pair.path, err)
}
}
return w.Close()
}
// NewForThisHost generates a new bootstrap file for the current host, based on
// the existing environment as well as data in garage.
func NewForThisHost(env *crypticnet.Env, into io.Writer) error {
client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
return newBootstrap(
env.Context,
into,
provideFromFS(env.Bootstrap.FS), // hostname
provideFromFS(env.Bootstrap.FS), // nebulaCertsCACert
provideFromFS(env.Bootstrap.FS), // nebulaCertsHostCert
provideFromFS(env.Bootstrap.FS), // nebulaCertsHostKey
provideDirFromGarage(client), // nebulaHosts
provideFromFS(env.Bootstrap.FS), // garageRPCSecret
provideFromFS(env.Bootstrap.FS), // garageGlobalBucketKey
provideDirFromGarage(client), // garageHosts
)
}
// NewForHost generates a new bootstrap file for an arbitrary host, based on the
// given admin file's FS and data in garage.
func NewForHost(env *crypticnet.Env, adminFS fs.FS, name string, into io.Writer) error {
host, ok := env.Bootstrap.Hosts[name]
if !ok {
return fmt.Errorf("unknown host %q, make sure host entry has been created", name)
}
client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
nebulaHostCert, err := nebula.NewHostCert(adminFS, host.Nebula)
if err != nil {
return fmt.Errorf("creating new nebula host key/cert: %w", err)
}
return newBootstrap(
env.Context,
into,
provideFromBytes([]byte(name)), // hostname
provideFromBytes(nebulaHostCert.CACert), // nebulaCertsCACert
provideFromBytes(nebulaHostCert.HostCert), // nebulaCertsHostCert
provideFromBytes(nebulaHostCert.HostKey), // nebulaCertsHostKey
provideDirFromGarage(client), // nebulaHosts
provideFromFS(adminFS), // garageRPCSecret
provideFromFS(adminFS), // garageGlobalBucketKey
provideDirFromGarage(client), // garageHosts
)
}

121
go-workspace/src/bootstrap/creator/provider.go
View File

@ -1,121 +0,0 @@
package creator
import (
"context"
"cryptic-net/garage"
"cryptic-net/tarutil"
"fmt"
"io/fs"
"github.com/minio/minio-go/v7"
)
// provider is a function which will populate the given filePath into the given
// TGZWriter. The path may be a file or a directory.
type provider func(context.Context, *tarutil.TGZWriter, string) error
func provideFromBytes(body []byte) provider {
return func(
ctx context.Context,
w *tarutil.TGZWriter,
filePath string,
) error {
w.WriteFileBytes(filePath, body)
return nil
}
}
func provideFromFS(srcFS fs.FS) provider {
return func(
ctx context.Context,
w *tarutil.TGZWriter,
filePath string,
) error {
return w.CopyFileFromFS(filePath, srcFS)
}
}
func provideDirFromFS(srcFS fs.FS) provider {
return func(
ctx context.Context,
w *tarutil.TGZWriter,
dirPath string,
) error {
return fs.WalkDir(
srcFS, dirPath,
func(filePath string, dirEntry fs.DirEntry, err error) error {
if err != nil {
return err
} else if dirEntry.IsDir() {
return nil
} else if err := w.CopyFileFromFS(filePath, srcFS); err != nil {
return fmt.Errorf("copying file %q: %w", filePath, err)
}
return nil
},
)
}
}
// TODO it'd be great if we could wrap a minio.Client into an fs.FS. That would
// get rid of a weird dependency in this package, and clean up this code a ton.
func provideDirFromGarage(client *minio.Client) provider {
return func(
ctx context.Context,
w *tarutil.TGZWriter,
dirPath string,
) error {
objInfoCh := client.ListObjects(
ctx, garage.GlobalBucket,
minio.ListObjectsOptions{
Prefix: dirPath,
Recursive: true,
},
)
for objInfo := range objInfoCh {
if objInfo.Err != nil {
return fmt.Errorf("listing objects: %w", objInfo.Err)
}
obj, err := client.GetObject(
ctx, garage.GlobalBucket, objInfo.Key, minio.GetObjectOptions{},
)
if err != nil {
return fmt.Errorf(
"retrieving object %q from global bucket: %w",
objInfo.Key, err,
)
}
objStat, err := obj.Stat()
if err != nil {
obj.Close()
return fmt.Errorf(
"stating object %q from global bucket: %w",
objInfo.Key, err,
)
}
w.WriteFile(objInfo.Key, objStat.Size, obj)
obj.Close()
}
return nil
}
}

48
go-workspace/src/bootstrap/garage.go
View File

@ -1,48 +0,0 @@
package bootstrap
import (
"cryptic-net/garage"
)
// Paths within the bootstrap FS related to garage.
const (
GarageGlobalBucketKeyYmlPath = "garage/global-bucket-key.yml"
GarageRPCSecretPath = "garage/rpc-secret.txt"
GarageHostsDirPath = "garage/hosts"
)
// GaragePeers returns a Peer for each known garage instance in the network.
func (b Bootstrap) GaragePeers() []garage.Peer {
var peers []garage.Peer
for _, host := range b.Hosts {
if host.Garage == nil {
continue
}
for _, instance := range host.Garage.Instances {
peer := garage.Peer{
IP: host.Nebula.IP,
RPCPort: instance.RPCPort,
S3APIPort: instance.S3APIPort,
}
peers = append(peers, peer)
}
}
return peers
}
// GarageRPCPeerAddrs returns the full RPC peer address for each known garage
// instance in the network.
func (b Bootstrap) GarageRPCPeerAddrs() []string {
var addrs []string
for _, peer := range b.GaragePeers() {
addrs = append(addrs, peer.RPCPeerAddr())
}
return addrs
}

10
go-workspace/src/bootstrap/nebula.go
View File

@ -1,10 +0,0 @@
package bootstrap
// Paths within the bootstrap FS related to nebula.
const (
NebulaHostsDirPath = "nebula/hosts"
NebulaCertsCACertPath = "nebula/certs/ca.crt"
NebulaCertsHostCertPath = "nebula/certs/host.crt"
NebulaCertsHostKeyPath = "nebula/certs/host.key"
)

161
go-workspace/src/bootstrap/new_for_host.go Normal file
View File

@ -0,0 +1,161 @@
package bootstrap
import (
crypticnet "cryptic-net"
"cryptic-net/garage"
"cryptic-net/tarutil"
"crypto/rand"
"fmt"
"io"
"io/fs"
"net"
"time"
"github.com/slackhq/nebula/cert"
"golang.org/x/crypto/curve25519"
)
var ipCIDRMask = func() net.IPMask {
_, ipNet, err := net.ParseCIDR("10.10.0.0/16")
if err != nil {
panic(err)
}
return ipNet.Mask
}()
// Generates a new key/cert for a nebula host, writing their encoded forms into
// the given TGZWriter. It will also write the ca.crt file to the TGZWriter.
//
// The logic here is largely based on
// https://github.com/slackhq/nebula/blob/v1.4.0/cmd/nebula-cert/sign.go
func writeNewNebulaCert(
w *tarutil.TGZWriter, adminFS fs.FS, host crypticnet.NebulaHost,
) error {
caKeyPEM, err := fs.ReadFile(adminFS, "nebula/certs/ca.key")
if err != nil {
return fmt.Errorf("reading ca.key from admin fs: %w", err)
}
caKey, _, err := cert.UnmarshalEd25519PrivateKey(caKeyPEM)
if err != nil {
return fmt.Errorf("unmarshaling ca.key: %w", err)
}
caCrtPEM, err := fs.ReadFile(adminFS, "nebula/certs/ca.crt")
if err != nil {
return fmt.Errorf("reading ca.crt from admin fs: %w", err)
}
caCrt, _, err := cert.UnmarshalNebulaCertificateFromPEM(caCrtPEM)
if err != nil {
return fmt.Errorf("unmarshaling ca.crt: %w", err)
}
issuer, err := caCrt.Sha256Sum()
if err != nil {
return fmt.Errorf("getting ca.crt issuer: %w", err)
}
expireAt := caCrt.Details.NotAfter.Add(-1 * time.Second)
ip := net.ParseIP(host.IP)
if ip == nil {
return fmt.Errorf("invalid host ip %q", host.IP)
}
ipNet := &net.IPNet{
IP: ip,
Mask: ipCIDRMask,
}
var hostPub, hostKey []byte
{
var pubkey, privkey [32]byte
if _, err := io.ReadFull(rand.Reader, privkey[:]); err != nil {
return fmt.Errorf("reading random bytes to form private key: %w", err)
}
curve25519.ScalarBaseMult(&pubkey, &privkey)
hostPub, hostKey = pubkey[:], privkey[:]
}
hostCrt := cert.NebulaCertificate{
Details: cert.NebulaCertificateDetails{
Name: host.Name,
Ips: []*net.IPNet{ipNet},
NotBefore: time.Now(),
NotAfter: expireAt,
PublicKey: hostPub,
IsCA: false,
Issuer: issuer,
},
}
if err := hostCrt.CheckRootConstrains(caCrt); err != nil {
return fmt.Errorf("validating certificate constraints: %w", err)
}
if err := hostCrt.Sign(caKey); err != nil {
return fmt.Errorf("signing host cert with ca.key: %w", err)
}
hostKeyPEM := cert.MarshalX25519PrivateKey(hostKey)
hostCrtPEM, err := hostCrt.MarshalToPEM()
if err != nil {
return fmt.Errorf("marshalling host.crt: %w", err)
}
w.WriteFileBytes("nebula/certs/ca.crt", caCrtPEM)
w.WriteFileBytes("nebula/certs/host.key", hostKeyPEM)
w.WriteFileBytes("nebula/certs/host.crt", hostCrtPEM)
return nil
}
// NewForHost generates a new bootstrap file for an arbitrary host, based on the
// given admin file's FS and data in garage.
func NewForHost(env *crypticnet.Env, adminFS fs.FS, name string, into io.Writer) error {
host, ok := env.Hosts[name]
if !ok {
return fmt.Errorf("unknown host %q, make sure host entry has been created", name)
}
client, err := garage.GlobalBucketAPIClient(env)
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
w := tarutil.NewTGZWriter(into)
w.WriteFileBytes("hostname", []byte(name))
if err := writeNewNebulaCert(w, adminFS, host.Nebula); err != nil {
return fmt.Errorf("creating/adding host's nebula certs: %w", err)
}
fsFilesToCopy := []string{
"garage/rpc-secret.txt",
"garage/cryptic-net-global-bucket-key.yml",
}
for _, filePath := range fsFilesToCopy {
if err := copyFSFile(w, adminFS, filePath); err != nil {
return fmt.Errorf("copying %q from bootstrap fs: %w", filePath, err)
}
}
garageDirsToCopy := []string{
"nebula/hosts",
"garage/hosts",
}
for _, dirPath := range garageDirsToCopy {
if err := copyGarageDir(env.Context, client, w, dirPath); err != nil {
return fmt.Errorf("copying %q from garage: %w", dirPath, err)
}
}
return w.Close()
}

49
go-workspace/src/bootstrap/new_for_this_host.go Normal file
View File

@ -0,0 +1,49 @@
package bootstrap
import (
crypticnet "cryptic-net"
"cryptic-net/garage"
"cryptic-net/tarutil"
"fmt"
"io"
)
// NewForThisHost generates a new bootstrap file for the current host, based on
// the existing environment as well as data in garage.
func NewForThisHost(env *crypticnet.Env, into io.Writer) error {
client, err := garage.GlobalBucketAPIClient(env)
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
w := tarutil.NewTGZWriter(into)
fsFilesToCopy := []string{
"hostname",
"nebula/certs/ca.crt",
"nebula/certs/host.crt",
"nebula/certs/host.key",
"garage/rpc-secret.txt",
"garage/cryptic-net-global-bucket-key.yml",
}
for _, filePath := range fsFilesToCopy {
if err := copyFSFile(w, env.BootstrapFS, filePath); err != nil {
return fmt.Errorf("copying %q from bootstrap fs: %w", filePath, err)
}
}
garageDirsToCopy := []string{
"nebula/hosts",
"garage/hosts",
}
for _, dirPath := range garageDirsToCopy {
if err := copyGarageDir(env.Context, client, w, dirPath); err != nil {
return fmt.Errorf("copying %q from garage: %w", dirPath, err)
}
}
return w.Close()
}

74
go-workspace/src/bootstrap/util.go Normal file
View File

@ -0,0 +1,74 @@
package bootstrap
import (
"context"
"cryptic-net/garage"
"cryptic-net/tarutil"
"fmt"
"io/fs"
"github.com/minio/minio-go/v7"
)
func copyFSFile(w *tarutil.TGZWriter, srcFS fs.FS, path string) error {
f, err := srcFS.Open(path)
if err != nil {
return fmt.Errorf("opening %q in bootstrap fs: %w", path, err)
}
defer f.Close()
fStat, err := f.Stat()
if err != nil {
return fmt.Errorf("stating %q from bootstrap fs: %w", path, err)
}
w.WriteFile(path, fStat.Size(), f)
return nil
}
func copyGarageDir(
ctx context.Context, client *minio.Client,
w *tarutil.TGZWriter, path string,
) error {
objInfoCh := client.ListObjects(
ctx, garage.GlobalBucket,
minio.ListObjectsOptions{
Prefix: path,
Recursive: true,
},
)
for objInfo := range objInfoCh {
if objInfo.Err != nil {
return fmt.Errorf("listing objects: %w", objInfo.Err)
}
obj, err := client.GetObject(
ctx, garage.GlobalBucket, objInfo.Key, minio.GetObjectOptions{},
)
if err != nil {
return fmt.Errorf(
"retrieving object %q from global bucket: %w",
objInfo.Key, err,
)
}
objStat, err := obj.Stat()
if err != nil {
obj.Close()
return fmt.Errorf(
"stating object %q from global bucket: %w",
objInfo.Key, err,
)
}
w.WriteFile(objInfo.Key, objStat.Size, obj)
obj.Close()
}
return nil
}

22
go-workspace/src/cmd/entrypoint/daemon.go
View File

@ -14,7 +14,6 @@ import (
crypticnet "cryptic-net" crypticnet "cryptic-net"
"cryptic-net/bootstrap" "cryptic-net/bootstrap"
bootstrap_creator "cryptic-net/bootstrap/creator"
"cryptic-net/yamlutil" "cryptic-net/yamlutil"
"github.com/cryptic-io/pmux/pmuxlib" "github.com/cryptic-io/pmux/pmuxlib"
@ -112,16 +111,21 @@ func reloadBootstrap(env *crypticnet.Env) (bool, error) {
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
if err := bootstrap_creator.NewForThisHost(env, buf); err != nil { if err := bootstrap.NewForThisHost(env, buf); err != nil {
return false, fmt.Errorf("generating new bootstrap from env: %w", err) return false, fmt.Errorf("generating new bootstrap from env: %w", err)
} }
newBootstrap, err := bootstrap.FromReader(bytes.NewReader(buf.Bytes())) newHash, err := bootstrap.GetHashFromReader(bytes.NewReader(buf.Bytes()))
if err != nil { if err != nil {
return false, fmt.Errorf("parsing bootstrap which was just created: %w", err) return false, fmt.Errorf("reading hash from new bootstrap file: %w", err)
} }
if bytes.Equal(newBootstrap.Hash, env.Bootstrap.Hash) { currHash, err := bootstrap.GetHashFromFS(env.BootstrapFS)
if err != nil {
return false, fmt.Errorf("reading hash from existing bootstrap fs: %w", err)
}
if bytes.Equal(newHash, currHash) {
return false, nil return false, nil
} }
@ -132,12 +136,12 @@ func reloadBootstrap(env *crypticnet.Env) (bool, error) {
return true, nil return true, nil
} }
// runs a single pmux process ofor daemon, returning only once the env.Context // runs a single pmux process for daemon, returning only once the env.Context
// has been canceled or bootstrap info has been changed. This will always block // has been canceled or bootstrap info has been changed. This will always block
// until the spawned pmux has returned. // until the spawned pmux has returned.
func runDaemonPmuxOnce(env *crypticnet.Env) error { func runDaemonPmuxOnce(env *crypticnet.Env) error {
thisHost := env.Bootstrap.ThisHost() thisHost := env.ThisHost()
fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP) fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP)
pmuxProcConfigs := []pmuxlib.ProcessConfig{ pmuxProcConfigs := []pmuxlib.ProcessConfig{
@ -153,7 +157,7 @@ func runDaemonPmuxOnce(env *crypticnet.Env) error {
Cmd: "bash", Cmd: "bash",
Args: []string{ Args: []string{
"wait-for-ip", "wait-for-ip",
thisHost.Nebula.IP, env.ThisHost().Nebula.IP,
"bash", "bash",
"dnsmasq-entrypoint", "dnsmasq-entrypoint",
}, },
@ -166,7 +170,7 @@ func runDaemonPmuxOnce(env *crypticnet.Env) error {
Cmd: "bash", Cmd: "bash",
Args: []string{ Args: []string{
"wait-for-ip", "wait-for-ip",
thisHost.Nebula.IP, env.ThisHost().Nebula.IP,
"cryptic-net-main", "garage-entrypoint", "cryptic-net-main", "garage-entrypoint",
}, },

52
go-workspace/src/cmd/entrypoint/garage.go
View File

@ -2,10 +2,31 @@ package entrypoint
import ( import (
"fmt" "fmt"
"io/fs"
"log"
"os" "os"
"strings"
"syscall" "syscall"
crypticnet "cryptic-net"
"cryptic-net/garage"
) )
func getGaragePeer(env *crypticnet.Env) (string, error) {
if allocs := env.ThisDaemon().Storage.Allocations; len(allocs) > 0 {
return garage.GeneratePeerAddr(env.ThisHost().Nebula.IP, allocs[0].RPCPort)
}
bootstrapPeers, err := garage.BootstrapPeerAddrs(env.Hosts)
if err != nil {
return "", err
}
return bootstrapPeers[0], nil
}
var subCmdGarageMC = subCmd{ var subCmdGarageMC = subCmd{
name: "mc", name: "mc",
descr: "Runs the mc (minio-client) binary. The cryptic-net garage can be accessed under the `garage` alias", descr: "Runs the mc (minio-client) binary. The cryptic-net garage can be accessed under the `garage` alias",
@ -30,16 +51,21 @@ var subCmdGarageMC = subCmd{
env := subCmdCtx.env env := subCmdCtx.env
s3APIAddr := env.ChooseGaragePeer().S3APIAddr() apiAddr := garage.APIAddr(env)
if *keyID == "" || *keySecret == "" { if *keyID == "" || *keySecret == "" {
globalBucketCreds, err := garage.GlobalBucketAPICredentials(env)
if err != nil {
return fmt.Errorf("loading global bucket credentials: %w", err)
}
if *keyID == "" { if *keyID == "" {
*keyID = env.Bootstrap.GarageGlobalBucketS3APICredentials.ID *keyID = globalBucketCreds.ID
} }
if *keySecret == "" { if *keySecret == "" {
*keyID = env.Bootstrap.GarageGlobalBucketS3APICredentials.Secret *keySecret = globalBucketCreds.Secret
} }
} }
@ -57,7 +83,7 @@ var subCmdGarageMC = subCmd{
os.Environ(), os.Environ(),
fmt.Sprintf( fmt.Sprintf(
"MC_HOST_garage=http://%s:%s@%s", "MC_HOST_garage=http://%s:%s@%s",
*keyID, *keySecret, s3APIAddr, *keyID, *keySecret, apiAddr,
), ),
// The garage docs say this is necessary, though nothing bad // The garage docs say this is necessary, though nothing bad
@ -85,13 +111,27 @@ var subCmdGarageCLI = subCmd{
env := subCmdCtx.env env := subCmdCtx.env
peerAddr, err := getGaragePeer(env)
if err != nil {
return fmt.Errorf("picking peer to communicate with: %w", err)
}
rpcSecretB, err := fs.ReadFile(env.BootstrapFS, "garage/rpc-secret.txt")
if err != nil {
log.Fatalf("reading garage rpc secret bootstrap fs: %v", err)
}
rpcSecret := strings.TrimSpace(string(rpcSecretB))
var ( var (
binPath = env.BinPath("garage") binPath = env.BinPath("garage")
args = append([]string{"garage"}, subCmdCtx.args...) args = append([]string{"garage"}, subCmdCtx.args...)
cliEnv = append( cliEnv = append(
os.Environ(), os.Environ(),
"GARAGE_RPC_HOST="+env.ChooseGaragePeer().RPCAddr(), "GARAGE_RPC_HOST="+peerAddr,
"GARAGE_RPC_SECRET="+env.Bootstrap.GarageRPCSecret, "GARAGE_RPC_SECRET="+rpcSecret,
) )
) )

14
go-workspace/src/cmd/entrypoint/hosts.go
View File

@ -2,8 +2,8 @@ package entrypoint
import ( import (
"bytes" "bytes"
crypticnet "cryptic-net"
"cryptic-net/bootstrap" "cryptic-net/bootstrap"
bootstrap_creator "cryptic-net/bootstrap/creator"
"cryptic-net/garage" "cryptic-net/garage"
"cryptic-net/tarutil" "cryptic-net/tarutil"
"errors" "errors"
@ -73,12 +73,12 @@ var subCmdHostsAdd = subCmd{
env := subCmdCtx.env env := subCmdCtx.env
client, err := env.GlobalBucketS3APIClient() client, err := garage.GlobalBucketAPIClient(env)
if err != nil { if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err) return fmt.Errorf("creating client for global bucket: %w", err)
} }
nebulaHost := bootstrap.NebulaHost{ nebulaHost := crypticnet.NebulaHost{
Name: *name, Name: *name,
IP: *ip, IP: *ip,
} }
@ -113,7 +113,7 @@ var subCmdHostsList = subCmd{
env := subCmdCtx.env env := subCmdCtx.env
client, err := env.GlobalBucketS3APIClient() client, err := garage.GlobalBucketAPIClient(env)
if err != nil { if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err) return fmt.Errorf("creating client for global bucket: %w", err)
} }
@ -147,7 +147,7 @@ var subCmdHostsList = subCmd{
return fmt.Errorf("retrieving object %q from global bucket: %w", objInfo.Key, err) return fmt.Errorf("retrieving object %q from global bucket: %w", objInfo.Key, err)
} }
var nebulaHost bootstrap.NebulaHost var nebulaHost crypticnet.NebulaHost
err = yaml.NewDecoder(obj).Decode(&nebulaHost) err = yaml.NewDecoder(obj).Decode(&nebulaHost)
obj.Close() obj.Close()
@ -191,7 +191,7 @@ var subCmdHostsDelete = subCmd{
filePath := nebulaHostPath(*name) filePath := nebulaHostPath(*name)
client, err := env.GlobalBucketS3APIClient() client, err := garage.GlobalBucketAPIClient(env)
if err != nil { if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err) return fmt.Errorf("creating client for global bucket: %w", err)
} }
@ -263,7 +263,7 @@ var subCmdHostsMakeBootstrap = subCmd{
return fmt.Errorf("reading admin.tgz with --admin-path of %q: %w", *adminPath, err) return fmt.Errorf("reading admin.tgz with --admin-path of %q: %w", *adminPath, err)
} }
return bootstrap_creator.NewForHost(subCmdCtx.env, adminFS, *name, os.Stdout) return bootstrap.NewForHost(subCmdCtx.env, adminFS, *name, os.Stdout)
}, },
} }

49
go-workspace/src/cmd/garage-entrypoint/main.go
View File

@ -2,11 +2,13 @@ package garage_entrypoint
import ( import (
"fmt" "fmt"
"io/fs"
"log" "log"
"net" "net"
"os" "os"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings"
"time" "time"
crypticnet "cryptic-net" crypticnet "cryptic-net"
@ -17,23 +19,24 @@ import (
func writeChildConf( func writeChildConf(
env *crypticnet.Env, env *crypticnet.Env,
bootstrapPeers []string,
alloc crypticnet.DaemonYmlStorageAllocation, alloc crypticnet.DaemonYmlStorageAllocation,
rpcSecret string,
) (string, error) { ) (string, error) {
if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil { if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err) return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
} }
thisHost := env.Bootstrap.ThisHost() pubKey, privKey, err := garage.GeneratePeerKey(env.ThisHost().Nebula.IP, alloc.RPCPort)
peer := garage.Peer{ if err != nil {
IP: thisHost.Nebula.IP, return "", fmt.Errorf(
RPCPort: alloc.RPCPort, "generating node key with input %q,%d: %w",
S3APIPort: alloc.S3APIPort, env.ThisHost().Nebula.IP, alloc.RPCPort, err,
)
} }
pubKey, privKey := peer.RPCPeerKey()
nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key") nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub") nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
@ -48,17 +51,17 @@ func writeChildConf(
env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort), env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
) )
err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{ err = garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
MetaPath: alloc.MetaPath, MetaPath: alloc.MetaPath,
DataPath: alloc.DataPath, DataPath: alloc.DataPath,
RPCSecret: env.Bootstrap.GarageRPCSecret, RPCSecret: rpcSecret,
RPCAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)), RPCAddr: net.JoinHostPort(env.ThisHost().Nebula.IP, strconv.Itoa(alloc.RPCPort)),
APIAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)), APIAddr: net.JoinHostPort(env.ThisHost().Nebula.IP, strconv.Itoa(alloc.APIPort)),
WebAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.WebPort)), WebAddr: net.JoinHostPort(env.ThisHost().Nebula.IP, strconv.Itoa(alloc.WebPort)),
BootstrapPeers: env.Bootstrap.GarageRPCPeerAddrs(), BootstrapPeers: bootstrapPeers,
}) })
if err != nil { if err != nil {
@ -70,15 +73,13 @@ func writeChildConf(
func waitForArgs(env *crypticnet.Env, bin string, binArgs ...string) []string { func waitForArgs(env *crypticnet.Env, bin string, binArgs ...string) []string {
thisHost := env.Bootstrap.ThisHost()
var args []string var args []string
for _, alloc := range env.ThisDaemon().Storage.Allocations { for _, alloc := range env.ThisDaemon().Storage.Allocations {
args = append( args = append(
args, args,
"wait-for", "wait-for",
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)), net.JoinHostPort(env.ThisHost().Nebula.IP, strconv.Itoa(alloc.RPCPort)),
"--", "--",
) )
} }
@ -97,11 +98,25 @@ func Main() {
log.Fatalf("reading envvars: %v", err) log.Fatalf("reading envvars: %v", err)
} }
bootstrapPeers, err := garage.BootstrapPeerAddrs(env.Hosts)
if err != nil {
log.Fatalf("generating set of bootstrap peers: %v", err)
}
rpcSecretB, err := fs.ReadFile(env.BootstrapFS, "garage/rpc-secret.txt")
if err != nil {
log.Fatalf("reading garage rpc secret bootstrap fs: %v", err)
}
rpcSecret := strings.TrimSpace(string(rpcSecretB))
var pmuxProcConfigs []pmuxlib.ProcessConfig var pmuxProcConfigs []pmuxlib.ProcessConfig
for _, alloc := range env.ThisDaemon().Storage.Allocations { for _, alloc := range env.ThisDaemon().Storage.Allocations {
childConfPath, err := writeChildConf(env, alloc) childConfPath, err := writeChildConf(env, bootstrapPeers, alloc, rpcSecret)
if err != nil { if err != nil {
log.Fatalf("writing child config file for alloc %+v: %v", alloc, err) log.Fatalf("writing child config file for alloc %+v: %v", alloc, err)

29
go-workspace/src/cmd/garage-layout-diff/main.go
View File

@ -148,30 +148,29 @@ func readCurrNodes(r io.Reader) (clusterNodes, int, error) {
return currNodes, version, nil return currNodes, version, nil
} }
func readExpNodes(env *crypticnet.Env) clusterNodes { func readExpNodes(env *crypticnet.Env) (clusterNodes, error) {
thisHost := env.Bootstrap.ThisHost()
var expNodes clusterNodes var expNodes clusterNodes
for _, alloc := range env.ThisDaemon().Storage.Allocations { for _, alloc := range env.ThisDaemon().Storage.Allocations {
peer := garage.Peer{ id, err := garage.GeneratePeerID(env.ThisHost().Nebula.IP, alloc.RPCPort)
IP: thisHost.Nebula.IP,
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
}
id := peer.RPCPeerID() if err != nil {
return nil, fmt.Errorf(
"generating peer id for ip:%q port:%d: %w",
env.ThisHost().Nebula.IP, alloc.RPCPort, err,
)
}
expNodes = append(expNodes, clusterNode{ expNodes = append(expNodes, clusterNode{
ID: id, ID: id,
Zone: env.Bootstrap.HostName, Zone: env.ThisHost().Name,
Capacity: alloc.Capacity / 100, Capacity: alloc.Capacity / 100,
}) })
} }
return expNodes return expNodes, nil
} }
// NOTE: The id formatting for currNodes and expNodes is different; expNodes has // NOTE: The id formatting for currNodes and expNodes is different; expNodes has
@ -233,14 +232,18 @@ func Main() {
for _, node := range currNodes { for _, node := range currNodes {
if env.Bootstrap.HostName != node.Zone { if env.ThisHost().Name != node.Zone {
continue continue
} }
thisCurrNodes = append(thisCurrNodes, node) thisCurrNodes = append(thisCurrNodes, node)
} }
expNodes := readExpNodes(env) expNodes, err := readExpNodes(env)
if err != nil {
panic(fmt.Errorf("reading expected layout from environment: %w", err))
}
lines := diff(thisCurrNodes, expNodes) lines := diff(thisCurrNodes, expNodes)

9
go-workspace/src/cmd/garage-peer-keygen/main.go
View File

@ -42,12 +42,11 @@ func Main() {
panic("The arguments -ip, -port, and -danger are required") panic("The arguments -ip, -port, and -danger are required")
} }
peer := garage.Peer{ pubKey, privKey, err := garage.GeneratePeerKey(*ip, *port)
IP: *ip,
RPCPort: *port,
}
pubKey, privKey := peer.RPCPeerKey() if err != nil {
panic(fmt.Errorf("GeneratePeerKey returned: %w", err))
}
fmt.Fprintln(os.Stdout, hex.EncodeToString(pubKey)) fmt.Fprintln(os.Stdout, hex.EncodeToString(pubKey))

14
go-workspace/src/cmd/garage-update-global-bucket/main.go
View File

@ -3,7 +3,6 @@ package garage_update_global_bucket
import ( import (
"bytes" "bytes"
crypticnet "cryptic-net" crypticnet "cryptic-net"
"cryptic-net/bootstrap"
"cryptic-net/garage" "cryptic-net/garage"
"fmt" "fmt"
"log" "log"
@ -17,15 +16,12 @@ func updateGlobalBucket(env *crypticnet.Env) error {
ctx := env.Context ctx := env.Context
client, err := env.GlobalBucketS3APIClient() client, err := garage.GlobalBucketAPIClient(env)
if err != nil { if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err) return fmt.Errorf("creating client for global bucket: %w", err)
} }
filePath := filepath.Join( filePath := filepath.Join("garage/hosts", env.ThisHost().Name+".yml")
"garage/hosts",
env.Bootstrap.HostName+".yml",
)
daemon := env.ThisDaemon() daemon := env.ThisDaemon()
@ -45,13 +41,13 @@ func updateGlobalBucket(env *crypticnet.Env) error {
return nil return nil
} }
var garageHost bootstrap.GarageHost var garageHost crypticnet.GarageHost
for _, alloc := range daemon.Storage.Allocations { for _, alloc := range daemon.Storage.Allocations {
garageHostInstance := bootstrap.GarageHostInstance{ garageHostInstance := crypticnet.GarageHostInstance{
APIPort: alloc.APIPort,
RPCPort: alloc.RPCPort, RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
WebPort: alloc.WebPort, WebPort: alloc.WebPort,
} }

27
go-workspace/src/cmd/nebula-entrypoint/main.go
View File

@ -2,6 +2,8 @@ package nebula_entrypoint
import ( import (
"cryptic-net/yamlutil" "cryptic-net/yamlutil"
"fmt"
"io/fs"
"log" "log"
"net" "net"
"path/filepath" "path/filepath"
@ -25,7 +27,7 @@ func Main() {
staticHostMap = map[string][]string{} staticHostMap = map[string][]string{}
) )
for _, host := range env.Bootstrap.Hosts { for _, host := range env.Hosts {
if host.Nebula.PublicAddr == "" { if host.Nebula.PublicAddr == "" {
continue continue
@ -35,11 +37,26 @@ func Main() {
staticHostMap[host.Nebula.IP] = []string{host.Nebula.PublicAddr} staticHostMap[host.Nebula.IP] = []string{host.Nebula.PublicAddr}
} }
readCertFile := func(name string) string {
if err != nil {
return ""
}
path := filepath.Join("nebula", "certs", name)
var b []byte
if b, err = fs.ReadFile(env.BootstrapFS, path); err != nil {
err = fmt.Errorf("reading %q from bootstrap fs: %w", path, err)
}
return string(b)
}
config := map[string]interface{}{ config := map[string]interface{}{
"pki": map[string]string{ "pki": map[string]string{
"ca": env.Bootstrap.NebulaCertsCACert, "ca": readCertFile("ca.crt"),
"cert": env.Bootstrap.NebulaCertsHostCert, "cert": readCertFile("host.crt"),
"key": env.Bootstrap.NebulaCertsHostKey, "key": readCertFile("host.key"),
}, },
"static_host_map": staticHostMap, "static_host_map": staticHostMap,
"punchy": map[string]bool{ "punchy": map[string]bool{
@ -93,7 +110,7 @@ func Main() {
firewallInbound = append( firewallInbound = append(
firewallInbound, firewallInbound,
crypticnet.ConfigFirewallRule{ crypticnet.ConfigFirewallRule{
Port: strconv.Itoa(alloc.S3APIPort), Port: strconv.Itoa(alloc.APIPort),
Proto: "tcp", Proto: "tcp",
Host: "any", Host: "any",
}, },

4
go-workspace/src/cmd/nebula-update-global-bucket/main.go
View File

@ -16,14 +16,14 @@ func updateGlobalBucket(env *crypticnet.Env) error {
ctx := env.Context ctx := env.Context
client, err := env.GlobalBucketS3APIClient() client, err := garage.GlobalBucketAPIClient(env)
if err != nil { if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err) return fmt.Errorf("creating client for global bucket: %w", err)
} }
daemon := env.ThisDaemon() daemon := env.ThisDaemon()
host := env.Bootstrap.ThisHost() host := env.ThisHost()
host.Nebula.Name = host.Name host.Nebula.Name = host.Name
host.Nebula.PublicAddr = daemon.VPN.PublicAddr host.Nebula.PublicAddr = daemon.VPN.PublicAddr

2
go-workspace/src/daemon_yml.go
View File

@ -31,7 +31,7 @@ type DaemonYmlStorageAllocation struct {
DataPath string `yaml:"data_path"` DataPath string `yaml:"data_path"`
MetaPath string `yaml:"meta_path"` MetaPath string `yaml:"meta_path"`
Capacity int `yaml:"capacity"` Capacity int `yaml:"capacity"`
S3APIPort int `yaml:"api_port"` // TODO fix field name here APIPort int `yaml:"api_port"`
RPCPort int `yaml:"rpc_port"` RPCPort int `yaml:"rpc_port"`
WebPort int `yaml:"web_port"` WebPort int `yaml:"web_port"`
} }

36
go-workspace/src/env.go
View File

@ -2,7 +2,7 @@ package crypticnet
import ( import (
"context" "context"
"cryptic-net/bootstrap" "cryptic-net/tarutil"
"cryptic-net/yamlutil" "cryptic-net/yamlutil"
"errors" "errors"
"fmt" "fmt"
@ -37,7 +37,9 @@ type Env struct {
// If NewEnv is called with bootstrapOptional, and a bootstrap file is not // If NewEnv is called with bootstrapOptional, and a bootstrap file is not
// found, then these fields will not be set. // found, then these fields will not be set.
BootstrapPath string BootstrapPath string
Bootstrap bootstrap.Bootstrap BootstrapFS fs.FS
Hosts map[string]Host
HostName string
thisDaemon DaemonYml thisDaemon DaemonYml
thisDaemonOnce sync.Once thisDaemonOnce sync.Once
@ -116,13 +118,32 @@ func (e *Env) DataDirBootstrapPath() string {
// and all derived fields based on that. // and all derived fields based on that.
func (e *Env) LoadBootstrap(path string) error { func (e *Env) LoadBootstrap(path string) error {
var err error var (
err error
if e.Bootstrap, err = bootstrap.FromFile(path); err != nil { // load all values into temp variables before setting the fields on Env,
return fmt.Errorf("parsing bootstrap.tgz at %q: %w", path, err) // so we don't leave it in an inconsistent state.
bootstrapFS fs.FS
hosts map[string]Host
hostNameB []byte
)
if bootstrapFS, err = tarutil.FSFromTGZFile(path); err != nil {
return fmt.Errorf("reading bootstrap file at %q: %w", e.BootstrapPath, err)
}
if hosts, err = LoadHosts(bootstrapFS); err != nil {
return fmt.Errorf("loading hosts info from bootstrap fs: %w", err)
}
if hostNameB, err = fs.ReadFile(bootstrapFS, "hostname"); err != nil {
return fmt.Errorf("loading hostname from bootstrap fs: %w", err)
} }
e.BootstrapPath = path e.BootstrapPath = path
e.BootstrapFS = bootstrapFS
e.Hosts = hosts
e.HostName = string(hostNameB)
return nil return nil
} }
@ -198,6 +219,11 @@ func (e *Env) init(bootstrapOptional bool) error {
return nil return nil
} }
// ThisHost is a shortcut for returning env.Hosts[env.HostName].
func (e *Env) ThisHost() Host {
return e.Hosts[e.HostName]
}
// ToMap returns the Env as a map of key/value strings. If this map is set into // ToMap returns the Env as a map of key/value strings. If this map is set into
// a process's environment, then that process can read it back using ReadEnv. // a process's environment, then that process can read it back using ReadEnv.
func (e *Env) ToMap() map[string]string { func (e *Env) ToMap() map[string]string {

42
go-workspace/src/garage.go
View File

@ -1,42 +0,0 @@
package crypticnet
import (
"cryptic-net/garage"
"fmt"
)
// ChooseGaragePeer returns a Peer for a garage instance from the network. It
// will prefer a garage instance on this particular host, if there is one, but
// will otherwise return a random endpoint.
func (env *Env) ChooseGaragePeer() garage.Peer {
if allocs := env.ThisDaemon().Storage.Allocations; len(allocs) > 0 {
return garage.Peer{
IP: env.Bootstrap.ThisHost().Nebula.IP,
RPCPort: allocs[0].RPCPort,
S3APIPort: allocs[0].S3APIPort,
}
}
for _, peer := range env.Bootstrap.GaragePeers() {
return peer
}
panic("no garage instances configured")
}
// GlobalBucketS3APIClient returns an S3 client pre-configured with access to
// the global bucket.
func (env *Env) GlobalBucketS3APIClient() (garage.S3APIClient, error) {
addr := env.ChooseGaragePeer().S3APIAddr()
creds := env.Bootstrap.GarageGlobalBucketS3APICredentials
client, err := garage.NewS3APIClient(addr, creds)
if err != nil {
return nil, fmt.Errorf("connecting to garage S3 API At %q: %w", addr, err)
}
return client, err
}

74
go-workspace/src/garage/client.go
View File

@ -1,7 +1,12 @@
package garage package garage
import ( import (
crypticnet "cryptic-net"
"cryptic-net/yamlutil"
"errors" "errors"
"fmt"
"net"
"strconv"
"github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/credentials"
@ -14,21 +19,74 @@ func IsKeyNotFound(err error) bool {
return errors.As(err, &mErr) && mErr.Code == "NoSuchKey" return errors.As(err, &mErr) && mErr.Code == "NoSuchKey"
} }
// S3APIClient is a client used to interact with garage's S3 API. // APICredentials describe data fields necessary for authenticating with a
type S3APIClient = *minio.Client // garage api endpoint.
type APICredentials struct {
// S3APICredentials describe data fields necessary for authenticating with a
// garage S3 API endpoint.
type S3APICredentials struct {
ID string `yaml:"id"` ID string `yaml:"id"`
Secret string `yaml:"secret"` Secret string `yaml:"secret"`
} }
// NewS3APIClient returns a minio client configured to use the given garage S3 API // GlobalBucketAPICredentials returns APICredentials for the global bucket.
func GlobalBucketAPICredentials(env *crypticnet.Env) (APICredentials, error) {
const path = "garage/cryptic-net-global-bucket-key.yml"
var creds APICredentials
if err := yamlutil.LoadYamlFSFile(&creds, env.BootstrapFS, path); err != nil {
return APICredentials{}, fmt.Errorf("loading %q from bootstrap fs: %w", path, err)
}
return creds, nil
}
// APIAddr returns the network address of a garage api endpoint in the network.
// It will prefer an endpoint on this particular host, if there is one, but will
// otherwise return a random endpoint.
func APIAddr(env *crypticnet.Env) string {
if allocs := env.ThisDaemon().Storage.Allocations; len(allocs) > 0 {
return net.JoinHostPort(
env.ThisHost().Nebula.IP,
strconv.Itoa(allocs[0].APIPort),
)
}
for _, host := range env.Hosts {
if host.Garage == nil || len(host.Garage.Instances) == 0 {
continue
}
return net.JoinHostPort(
host.Nebula.IP,
strconv.Itoa(host.Garage.Instances[0].APIPort),
)
}
panic("no garage instances configured")
}
// APIClient returns a minio client configured to use the given garage API
// endpoint. // endpoint.
func NewS3APIClient(addr string, creds S3APICredentials) (S3APIClient, error) { func APIClient(addr string, creds APICredentials) (*minio.Client, error) {
return minio.New(addr, &minio.Options{ return minio.New(addr, &minio.Options{
Creds: credentials.NewStaticV4(creds.ID, creds.Secret, ""), Creds: credentials.NewStaticV4(creds.ID, creds.Secret, ""),
Region: Region, Region: Region,
}) })
} }
// GlobalBucketAPIClient returns a minio client pre-configured with access to
// the global bucket.
func GlobalBucketAPIClient(env *crypticnet.Env) (*minio.Client, error) {
creds, err := GlobalBucketAPICredentials(env)
if err != nil {
return nil, fmt.Errorf("loading global bucket credentials: %w", err)
}
addr := APIAddr(env)
return APIClient(addr, creds)
}

13
go-workspace/src/garage/garage.go
View File

@ -1,13 +0,0 @@
// Package garage contains helper functions and types which are useful for
// setting up garage configs, processes, and deployments.
package garage
const (
// Region is the region which garage is configured with.
Region = "garage"
// GlobalBucket is the name of the global garage bucket which is
// accessible to all hosts in the network.
GlobalBucket = "cryptic-net-global"
)

100
go-workspace/src/garage/garageutil.go Normal file
View File

@ -0,0 +1,100 @@
// Package garage contains helper functions and types which are useful for
// setting up garage configs, processes, and deployments.
package garage
import (
crypticnet "cryptic-net"
"crypto/ed25519"
"encoding/hex"
"fmt"
"net"
"strconv"
)
const (
// Region is the region which garage is configured with.
Region = "garage"
// GlobalBucket is the name of the global garage bucket which is
// accessible to all hosts in the network.
GlobalBucket = "cryptic-net-global"
)
// GeneratePeerKey deterministically generates a public/private keys which can
// be used as a garage node key.
//
// DANGER: This function will deterministically produce public/private keys
// given some arbitrary input. This is NEVER what you want. It's only being used
// in cryptic-net for a very specific purpose for which I think it's ok and is
// very necessary, and people are probably _still_ going to yell at me.
//
func GeneratePeerKey(ip string, port int) (pubKey, privKey []byte, err error) {
input := []byte(net.JoinHostPort(ip, strconv.Itoa(port)))
// Append the length of the input to the input, so that the input "foo"
// doesn't generate the same key as the input "foofoo".
input = strconv.AppendInt(input, int64(len(input)), 10)
return ed25519.GenerateKey(NewInfiniteReader(input))
}
// GeneratePeerID generates the peer id for the given peer.
//
// DANGER: See warning on GenerateNodeKey.
func GeneratePeerID(ip string, port int) (string, error) {
peerNodeKeyPub, _, err := GeneratePeerKey(ip, port)
if err != nil {
return "", err
}
return hex.EncodeToString(peerNodeKeyPub), nil
}
// GeneratePeerAddr generates the peer address (e.g. "id@ip:port") for the
// given peer.
//
// DANGER: See warning on GenerateNodeKey.
func GeneratePeerAddr(ip string, port int) (string, error) {
id, err := GeneratePeerID(ip, port)
if err != nil {
return "", fmt.Errorf("generating peer id: %w", err)
}
return fmt.Sprintf("%s@%s", id, net.JoinHostPort(ip, strconv.Itoa(port))), nil
}
// BootstrapPeerAddrs generates the list of bootstrap peer strings based on the
// bootstrap hosts.
func BootstrapPeerAddrs(hosts map[string]crypticnet.Host) ([]string, error) {
var peers []string
for _, host := range hosts {
if host.Garage == nil {
continue
}
for _, instance := range host.Garage.Instances {
peer, err := GeneratePeerAddr(host.Nebula.IP, instance.RPCPort)
if err != nil {
return nil, fmt.Errorf(
"generating peer address with input %q,%d: %w",
host.Nebula.IP, instance.RPCPort, err,
)
}
peers = append(peers, peer)
}
}
return peers, nil
}

66
go-workspace/src/garage/peer.go
View File

@ -1,66 +0,0 @@
package garage
import (
"crypto/ed25519"
"encoding/hex"
"fmt"
"net"
"strconv"
)
// Peer describes all information necessary to connect to a given garage node.
type Peer struct {
IP string
RPCPort int
S3APIPort int
}
// RPCPeerKey deterministically generates a public/private keys which can
// be used as a garage node key.
//
// DANGER: This function will deterministically produce public/private keys
// given some arbitrary input. This is NEVER what you want. It's only being used
// in cryptic-net for a very specific purpose for which I think it's ok and is
// very necessary, and people are probably _still_ going to yell at me.
//
func (p Peer) RPCPeerKey() (pubKey, privKey []byte) {
input := []byte(net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort)))
// Append the length of the input to the input, so that the input "foo"
// doesn't generate the same key as the input "foofoo".
input = strconv.AppendInt(input, int64(len(input)), 10)
pubKey, privKey, err := ed25519.GenerateKey(NewInfiniteReader(input))
if err != nil {
panic(err)
}
return pubKey, privKey
}
// RPCPeerID returns the peer ID of the garage node for use in communicating
// over RPC.
//
// DANGER: See warning on RPCPeerKey.
func (p Peer) RPCPeerID() string {
pubKey, _ := p.RPCPeerKey()
return hex.EncodeToString(pubKey)
}
// RPCAddr returns the address of the peer's RPC port.
func (p Peer) RPCAddr() string {
return net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort))
}
// RPCPeerAddr returns the full peer address (e.g. "id@ip:port") of the garage
// node for use in communicating over RPC.
//
// DANGER: See warning on RPCPeerKey.
func (p Peer) RPCPeerAddr() string {
return fmt.Sprintf("%s@%s", p.RPCPeerID(), p.RPCAddr())
}
// S3APIAddr returns the address of the peer's S3 API port.
func (p Peer) S3APIAddr() string {
return net.JoinHostPort(p.IP, strconv.Itoa(p.S3APIPort))
}

15
go-workspace/src/bootstrap/hosts.go → go-workspace/src/hosts.go
View File

@ -1,4 +1,4 @@
package bootstrap package crypticnet
import ( import (
"errors" "errors"
@ -19,8 +19,8 @@ type NebulaHost struct {
// GarageHostInstance describes a single garage instance running on a host. // GarageHostInstance describes a single garage instance running on a host.
type GarageHostInstance struct { type GarageHostInstance struct {
APIPort int `yaml:"api_port"`
RPCPort int `yaml:"rpc_port"` RPCPort int `yaml:"rpc_port"`
S3APIPort int `yaml:"s3_api_port"`
WebPort int `yaml:"web_port"` WebPort int `yaml:"web_port"`
} }
@ -37,7 +37,8 @@ type Host struct {
Garage *GarageHost Garage *GarageHost
} }
func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) { // LostHosts returns a mapping of hostnames to Host objects for each host.
func LoadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
hosts := map[string]Host{} hosts := map[string]Host{}
@ -51,11 +52,9 @@ func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
} }
{ {
globPath := filepath.Join(NebulaHostsDirPath, "*.yml") nebulaHostFiles, err := fs.Glob(bootstrapFS, "nebula/hosts/*.yml")
nebulaHostFiles, err := fs.Glob(bootstrapFS, globPath)
if err != nil { if err != nil {
return nil, fmt.Errorf("listing nebula host files at %q in fs: %w", globPath, err) return nil, fmt.Errorf("listing nebula host files: %w", err)
} }
for _, nebulaHostPath := range nebulaHostFiles { for _, nebulaHostPath := range nebulaHostFiles {
@ -77,7 +76,7 @@ func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
for hostName, host := range hosts { for hostName, host := range hosts {
garageHostPath := filepath.Join(GarageHostsDirPath, hostName+".yml") garageHostPath := filepath.Join("garage/hosts", hostName+".yml")
var garageHost GarageHost var garageHost GarageHost
if err := readAsYaml(&garageHost, garageHostPath); errors.Is(err, fs.ErrNotExist) { if err := readAsYaml(&garageHost, garageHostPath); errors.Is(err, fs.ErrNotExist) {

172
go-workspace/src/nebula/nebula.go
View File

@ -1,172 +0,0 @@
// Package nebula contains helper functions and types which are useful for
// setting up nebula configs, processes, and deployments.
package nebula
import (
"cryptic-net/bootstrap"
"crypto/ed25519"
"crypto/rand"
"fmt"
"io"
"io/fs"
"net"
"time"
"github.com/slackhq/nebula/cert"
"golang.org/x/crypto/curve25519"
)
// TODO this should one day not be hardcoded
var ipCIDRMask = func() net.IPMask {
_, ipNet, err := net.ParseCIDR("10.10.0.0/16")
if err != nil {
panic(err)
}
return ipNet.Mask
}()
// HostCert contains the certificate and private key files which will need to
// be present on a particular host. Each file is PEM encoded.
type HostCert struct {
CACert []byte
HostKey []byte
HostCert []byte
}
// CACert contains the certificate and private files which can be used to create
// HostCerts. Each file is PEM encoded.
type CACert struct {
CACert []byte
CAKey []byte
}
// NewHostCert generates a new key/cert for a nebula host using the CA key
// which will be found in the adminFS.
func NewHostCert(
adminFS fs.FS, host bootstrap.NebulaHost,
) (
HostCert, error,
) {
// The logic here is largely based on
// https://github.com/slackhq/nebula/blob/v1.4.0/cmd/nebula-cert/sign.go
caKeyPEM, err := fs.ReadFile(adminFS, "nebula/certs/ca.key")
if err != nil {
return HostCert{}, fmt.Errorf("reading ca.key from admin fs: %w", err)
}
caKey, _, err := cert.UnmarshalEd25519PrivateKey(caKeyPEM)
if err != nil {
return HostCert{}, fmt.Errorf("unmarshaling ca.key: %w", err)
}
caCrtPEM, err := fs.ReadFile(adminFS, "nebula/certs/ca.crt")
if err != nil {
return HostCert{}, fmt.Errorf("reading ca.crt from admin fs: %w", err)
}
caCrt, _, err := cert.UnmarshalNebulaCertificateFromPEM(caCrtPEM)
if err != nil {
return HostCert{}, fmt.Errorf("unmarshaling ca.crt: %w", err)
}
issuer, err := caCrt.Sha256Sum()
if err != nil {
return HostCert{}, fmt.Errorf("getting ca.crt issuer: %w", err)
}
expireAt := caCrt.Details.NotAfter.Add(-1 * time.Second)
ip := net.ParseIP(host.IP)
if ip == nil {
return HostCert{}, fmt.Errorf("invalid host ip %q", host.IP)
}
ipNet := &net.IPNet{
IP: ip,
Mask: ipCIDRMask,
}
var hostPub, hostKey []byte
{
var pubkey, privkey [32]byte
if _, err := io.ReadFull(rand.Reader, privkey[:]); err != nil {
return HostCert{}, fmt.Errorf("reading random bytes to form private key: %w", err)
}
curve25519.ScalarBaseMult(&pubkey, &privkey)
hostPub, hostKey = pubkey[:], privkey[:]
}
hostCrt := cert.NebulaCertificate{
Details: cert.NebulaCertificateDetails{
Name: host.Name,
Ips: []*net.IPNet{ipNet},
NotBefore: time.Now(),
NotAfter: expireAt,
PublicKey: hostPub,
IsCA: false,
Issuer: issuer,
},
}
if err := hostCrt.CheckRootConstrains(caCrt); err != nil {
return HostCert{}, fmt.Errorf("validating certificate constraints: %w", err)
}
if err := hostCrt.Sign(caKey); err != nil {
return HostCert{}, fmt.Errorf("signing host cert with ca.key: %w", err)
}
hostKeyPEM := cert.MarshalX25519PrivateKey(hostKey)
hostCrtPEM, err := hostCrt.MarshalToPEM()
if err != nil {
return HostCert{}, fmt.Errorf("marshalling host.crt: %w", err)
}
return HostCert{
CACert: caCrtPEM,
HostKey: hostKeyPEM,
HostCert: hostCrtPEM,
}, nil
}
// NewCACert generates a CACert. The domain should be the network's root domain,
// and is included in the signing certificate's Name field.
func NewCACert(domain string) (CACert, error) {
pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
panic(fmt.Errorf("generating ed25519 key: %w", err))
}
now := time.Now()
expireAt := now.Add(2 * 365 * 24 * time.Hour)
caCrt := cert.NebulaCertificate{
Details: cert.NebulaCertificateDetails{
Name: fmt.Sprintf("%s cryptic-net root cert", domain),
NotBefore: now,
NotAfter: expireAt,
PublicKey: pubKey,
IsCA: true,
},
}
if err := caCrt.Sign(privKey); err != nil {
return CACert{}, fmt.Errorf("signing caCrt: %w", err)
}
caKeyPEM := cert.MarshalEd25519PrivateKey(privKey)
caCrtPem, err := caCrt.MarshalToPEM()
if err != nil {
return CACert{}, fmt.Errorf("marshaling caCrt: %w", err)
}
return CACert{
CACert: caCrtPem,
CAKey: caKeyPEM,
}, nil
}

15
go-workspace/src/tarutil/tarutil.go
View File

@ -7,11 +7,12 @@ import (
"fmt" "fmt"
"io" "io"
"io/fs" "io/fs"
"os"
"github.com/nlepage/go-tarfs" "github.com/nlepage/go-tarfs"
) )
// FSFromReader returns a FS instance which will read the contents of a tgz // FSFromTGZFile returns a FS instance which will read the contents of a tgz
// file from the given Reader. // file from the given Reader.
func FSFromReader(r io.Reader) (fs.FS, error) { func FSFromReader(r io.Reader) (fs.FS, error) {
gf, err := gzip.NewReader(r) gf, err := gzip.NewReader(r)
@ -22,3 +23,15 @@ func FSFromReader(r io.Reader) (fs.FS, error) {
return tarfs.New(gf) return tarfs.New(gf)
} }
// FSFromTGZFile returns a FS instance which will read the contents of a tgz
// file.
func FSFromTGZFile(path string) (fs.FS, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("opening file: %w", err)
}
defer f.Close()
return FSFromReader(f)
}

20
go-workspace/src/tarutil/tgz_writer.go
View File

@ -7,7 +7,6 @@ import (
"crypto/sha512" "crypto/sha512"
"fmt" "fmt"
"io" "io"
"io/fs"
"path/filepath" "path/filepath"
"sort" "sort"
"strings" "strings"
@ -150,22 +149,3 @@ func (w *TGZWriter) WriteFileBytes(path string, body []byte) {
bodyR := bytes.NewReader(body) bodyR := bytes.NewReader(body)
w.WriteFile(path, bodyR.Size(), bodyR) w.WriteFile(path, bodyR.Size(), bodyR)
} }
// CopyFileFromFS copies the file at the given path from srcFS into the same
// path in the TGZWriter.
func (w *TGZWriter) CopyFileFromFS(path string, srcFS fs.FS) error {
f, err := srcFS.Open(path)
if err != nil {
return fmt.Errorf("opening: %w", err)
}
defer f.Close()
fStat, err := f.Stat()
if err != nil {
return fmt.Errorf("stating: %w", err)
}
w.WriteFile(path, fStat.Size(), f)
return nil
}