Compare commits

..

No commits in common. "836e69735d71e96043d1ca46b45daf72e9503c47" and "24b7fe63394d6f2183c62b3c12acba4faf36a181" have entirely different histories.

28 changed files with 671 additions and 815 deletions

View File

@ -2,129 +2,33 @@
package bootstrap package bootstrap
import ( import (
"cryptic-net/garage"
"cryptic-net/tarutil" "cryptic-net/tarutil"
"cryptic-net/yamlutil"
"fmt" "fmt"
"io" "io"
"io/fs" "io/fs"
"os"
"strings"
) )
// Paths within the bootstrap FS which for general data. // GetHashFromFS returns the hash of the contents of the given bootstrap file.
const ( // It may return nil if the bootstrap file doesn't have a hash.
HostNamePath = "hostname" func GetHashFromFS(bootstrapFS fs.FS) ([]byte, error) {
)
// Bootstrap is used for accessing all information contained within a b, err := fs.ReadFile(bootstrapFS, tarutil.HashBinPath)
// bootstrap.tgz file.
//
// An instance of Bootstrap is read-only, the creator sub-package should be used
// to create new instances.
type Bootstrap struct {
Hosts map[string]Host
HostName string
NebulaCertsCACert string
NebulaCertsHostCert string
NebulaCertsHostKey string
GarageRPCSecret string
GarageGlobalBucketS3APICredentials garage.S3APICredentials
// Hash is a determinstic hash of the contents of the bootstrap file. This
// will be populated when parsing a Bootstrap from a bootstrap.tgz, but will
// be ignored when creating a new bootstrap.tgz.
Hash []byte
// DEPRECATED do not use
FS fs.FS
}
// FromFS loads a Boostrap instance from the given fs.FS, which presumably
// represents the file structure of a bootstrap.tgz file.
func FromFS(bootstrapFS fs.FS) (Bootstrap, error) {
var (
b Bootstrap
err error
)
b.FS = bootstrapFS
if b.Hosts, err = loadHosts(bootstrapFS); err != nil {
return Bootstrap{}, fmt.Errorf("loading hosts info from fs: %w", err)
}
if err = yamlutil.LoadYamlFSFile(
&b.GarageGlobalBucketS3APICredentials,
bootstrapFS,
GarageGlobalBucketKeyYmlPath,
); err != nil {
return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", b.GarageGlobalBucketS3APICredentials, err)
}
filesToLoadAsString := []struct {
into *string
path string
}{
{&b.HostName, HostNamePath},
{&b.NebulaCertsCACert, NebulaCertsCACertPath},
{&b.NebulaCertsHostCert, NebulaCertsHostCertPath},
{&b.NebulaCertsHostKey, NebulaCertsHostKeyPath},
{&b.GarageRPCSecret, GarageRPCSecretPath},
}
for _, f := range filesToLoadAsString {
body, err := fs.ReadFile(bootstrapFS, f.path)
if err != nil { if err != nil {
return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", f.path, err) return nil, fmt.Errorf("reading file %q from bootstrap fs: %w", tarutil.HashBinPath, err)
}
*f.into = string(body)
}
// TODO confirm if this is necessary
b.GarageRPCSecret = strings.TrimSpace(b.GarageRPCSecret)
if b.Hash, err = fs.ReadFile(bootstrapFS, tarutil.HashBinPath); err != nil {
return Bootstrap{}, fmt.Errorf("loading %q from fs: %w", tarutil.HashBinPath, err)
} }
return b, nil return b, nil
} }
// FromReader reads a bootstrap.tgz file from the given io.Reader. // GetHashFromReader reads the given tgz file as an fs.FS, and passes that to
func FromReader(r io.Reader) (Bootstrap, error) { // GetHashFromFS.
func GetHashFromReader(r io.Reader) ([]byte, error) {
fs, err := tarutil.FSFromReader(r) bootstrapFS, err := tarutil.FSFromReader(r)
if err != nil { if err != nil {
return Bootstrap{}, fmt.Errorf("reading bootstrap.tgz: %w", err) return nil, fmt.Errorf("reading tar fs from reader: %w", err)
} }
return FromFS(fs) return GetHashFromFS(bootstrapFS)
}
// FromFile reads a bootstrap.tgz from a file at the given path.
func FromFile(path string) (Bootstrap, error) {
f, err := os.Open(path)
if err != nil {
return Bootstrap{}, fmt.Errorf("opening file: %w", err)
}
defer f.Close()
return FromReader(f)
}
// ThisHost is a shortcut for b.Hosts[b.HostName], but will panic if the
// HostName isn't found in the Hosts map.
func (b Bootstrap) ThisHost() Host {
host, ok := b.Hosts[b.HostName]
if !ok {
panic(fmt.Sprintf("hostname %q not defined in bootstrap's hosts", b.HostName))
}
return host
} }

View File

@ -1,110 +0,0 @@
// Package creator is responsible for creating bootstrap files. It exists
// separately from the main bootstrap package in order to prevent import loops
// due to its use of crypticnet.Env.
package creator
import (
"context"
crypticnet "cryptic-net"
"cryptic-net/bootstrap"
"cryptic-net/nebula"
"cryptic-net/tarutil"
"fmt"
"io"
"io/fs"
)
func newBootstrap(
ctx context.Context,
into io.Writer,
hostname provider,
nebulaCertsCACert provider,
nebulaCertsHostCert provider,
nebulaCertsHostKey provider,
nebulaHosts provider,
garageRPCSecret provider,
garageGlobalBucketKey provider,
garageHosts provider,
) error {
pairs := []struct {
path string
provider provider
}{
{bootstrap.HostNamePath, hostname},
{bootstrap.NebulaCertsCACertPath, nebulaCertsCACert},
{bootstrap.NebulaCertsHostCertPath, nebulaCertsHostCert},
{bootstrap.NebulaCertsHostKeyPath, nebulaCertsHostKey},
{bootstrap.NebulaHostsDirPath, nebulaHosts},
{bootstrap.GarageRPCSecretPath, garageRPCSecret},
{bootstrap.GarageGlobalBucketKeyYmlPath, garageGlobalBucketKey},
{bootstrap.GarageHostsDirPath, garageHosts},
}
w := tarutil.NewTGZWriter(into)
for _, pair := range pairs {
if err := pair.provider(ctx, w, pair.path); err != nil {
return fmt.Errorf("populating %q in new bootstrap: %w", pair.path, err)
}
}
return w.Close()
}
// NewForThisHost generates a new bootstrap file for the current host, based on
// the existing environment as well as data in garage.
func NewForThisHost(env *crypticnet.Env, into io.Writer) error {
client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
return newBootstrap(
env.Context,
into,
provideFromFS(env.Bootstrap.FS), // hostname
provideFromFS(env.Bootstrap.FS), // nebulaCertsCACert
provideFromFS(env.Bootstrap.FS), // nebulaCertsHostCert
provideFromFS(env.Bootstrap.FS), // nebulaCertsHostKey
provideDirFromGarage(client), // nebulaHosts
provideFromFS(env.Bootstrap.FS), // garageRPCSecret
provideFromFS(env.Bootstrap.FS), // garageGlobalBucketKey
provideDirFromGarage(client), // garageHosts
)
}
// NewForHost generates a new bootstrap file for an arbitrary host, based on the
// given admin file's FS and data in garage.
func NewForHost(env *crypticnet.Env, adminFS fs.FS, name string, into io.Writer) error {
host, ok := env.Bootstrap.Hosts[name]
if !ok {
return fmt.Errorf("unknown host %q, make sure host entry has been created", name)
}
client, err := env.GlobalBucketS3APIClient()
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
nebulaHostCert, err := nebula.NewHostCert(adminFS, host.Nebula)
if err != nil {
return fmt.Errorf("creating new nebula host key/cert: %w", err)
}
return newBootstrap(
env.Context,
into,
provideFromBytes([]byte(name)), // hostname
provideFromBytes(nebulaHostCert.CACert), // nebulaCertsCACert
provideFromBytes(nebulaHostCert.HostCert), // nebulaCertsHostCert
provideFromBytes(nebulaHostCert.HostKey), // nebulaCertsHostKey
provideDirFromGarage(client), // nebulaHosts
provideFromFS(adminFS), // garageRPCSecret
provideFromFS(adminFS), // garageGlobalBucketKey
provideDirFromGarage(client), // garageHosts
)
}

View File

@ -1,121 +0,0 @@
package creator
import (
"context"
"cryptic-net/garage"
"cryptic-net/tarutil"
"fmt"
"io/fs"
"github.com/minio/minio-go/v7"
)
// provider is a function which will populate the given filePath into the given
// TGZWriter. The path may be a file or a directory.
type provider func(context.Context, *tarutil.TGZWriter, string) error
func provideFromBytes(body []byte) provider {
return func(
ctx context.Context,
w *tarutil.TGZWriter,
filePath string,
) error {
w.WriteFileBytes(filePath, body)
return nil
}
}
func provideFromFS(srcFS fs.FS) provider {
return func(
ctx context.Context,
w *tarutil.TGZWriter,
filePath string,
) error {
return w.CopyFileFromFS(filePath, srcFS)
}
}
func provideDirFromFS(srcFS fs.FS) provider {
return func(
ctx context.Context,
w *tarutil.TGZWriter,
dirPath string,
) error {
return fs.WalkDir(
srcFS, dirPath,
func(filePath string, dirEntry fs.DirEntry, err error) error {
if err != nil {
return err
} else if dirEntry.IsDir() {
return nil
} else if err := w.CopyFileFromFS(filePath, srcFS); err != nil {
return fmt.Errorf("copying file %q: %w", filePath, err)
}
return nil
},
)
}
}
// TODO it'd be great if we could wrap a minio.Client into an fs.FS. That would
// get rid of a weird dependency in this package, and clean up this code a ton.
func provideDirFromGarage(client *minio.Client) provider {
return func(
ctx context.Context,
w *tarutil.TGZWriter,
dirPath string,
) error {
objInfoCh := client.ListObjects(
ctx, garage.GlobalBucket,
minio.ListObjectsOptions{
Prefix: dirPath,
Recursive: true,
},
)
for objInfo := range objInfoCh {
if objInfo.Err != nil {
return fmt.Errorf("listing objects: %w", objInfo.Err)
}
obj, err := client.GetObject(
ctx, garage.GlobalBucket, objInfo.Key, minio.GetObjectOptions{},
)
if err != nil {
return fmt.Errorf(
"retrieving object %q from global bucket: %w",
objInfo.Key, err,
)
}
objStat, err := obj.Stat()
if err != nil {
obj.Close()
return fmt.Errorf(
"stating object %q from global bucket: %w",
objInfo.Key, err,
)
}
w.WriteFile(objInfo.Key, objStat.Size, obj)
obj.Close()
}
return nil
}
}

View File

@ -1,48 +0,0 @@
package bootstrap
import (
"cryptic-net/garage"
)
// Paths within the bootstrap FS related to garage.
const (
GarageGlobalBucketKeyYmlPath = "garage/global-bucket-key.yml"
GarageRPCSecretPath = "garage/rpc-secret.txt"
GarageHostsDirPath = "garage/hosts"
)
// GaragePeers returns a Peer for each known garage instance in the network.
func (b Bootstrap) GaragePeers() []garage.Peer {
var peers []garage.Peer
for _, host := range b.Hosts {
if host.Garage == nil {
continue
}
for _, instance := range host.Garage.Instances {
peer := garage.Peer{
IP: host.Nebula.IP,
RPCPort: instance.RPCPort,
S3APIPort: instance.S3APIPort,
}
peers = append(peers, peer)
}
}
return peers
}
// GarageRPCPeerAddrs returns the full RPC peer address for each known garage
// instance in the network.
func (b Bootstrap) GarageRPCPeerAddrs() []string {
var addrs []string
for _, peer := range b.GaragePeers() {
addrs = append(addrs, peer.RPCPeerAddr())
}
return addrs
}

View File

@ -1,10 +0,0 @@
package bootstrap
// Paths within the bootstrap FS related to nebula.
const (
NebulaHostsDirPath = "nebula/hosts"
NebulaCertsCACertPath = "nebula/certs/ca.crt"
NebulaCertsHostCertPath = "nebula/certs/host.crt"
NebulaCertsHostKeyPath = "nebula/certs/host.key"
)

View File

@ -0,0 +1,161 @@
package bootstrap
import (
crypticnet "cryptic-net"
"cryptic-net/garage"
"cryptic-net/tarutil"
"crypto/rand"
"fmt"
"io"
"io/fs"
"net"
"time"
"github.com/slackhq/nebula/cert"
"golang.org/x/crypto/curve25519"
)
var ipCIDRMask = func() net.IPMask {
_, ipNet, err := net.ParseCIDR("10.10.0.0/16")
if err != nil {
panic(err)
}
return ipNet.Mask
}()
// Generates a new key/cert for a nebula host, writing their encoded forms into
// the given TGZWriter. It will also write the ca.crt file to the TGZWriter.
//
// The logic here is largely based on
// https://github.com/slackhq/nebula/blob/v1.4.0/cmd/nebula-cert/sign.go
func writeNewNebulaCert(
w *tarutil.TGZWriter, adminFS fs.FS, host crypticnet.NebulaHost,
) error {
caKeyPEM, err := fs.ReadFile(adminFS, "nebula/certs/ca.key")
if err != nil {
return fmt.Errorf("reading ca.key from admin fs: %w", err)
}
caKey, _, err := cert.UnmarshalEd25519PrivateKey(caKeyPEM)
if err != nil {
return fmt.Errorf("unmarshaling ca.key: %w", err)
}
caCrtPEM, err := fs.ReadFile(adminFS, "nebula/certs/ca.crt")
if err != nil {
return fmt.Errorf("reading ca.crt from admin fs: %w", err)
}
caCrt, _, err := cert.UnmarshalNebulaCertificateFromPEM(caCrtPEM)
if err != nil {
return fmt.Errorf("unmarshaling ca.crt: %w", err)
}
issuer, err := caCrt.Sha256Sum()
if err != nil {
return fmt.Errorf("getting ca.crt issuer: %w", err)
}
expireAt := caCrt.Details.NotAfter.Add(-1 * time.Second)
ip := net.ParseIP(host.IP)
if ip == nil {
return fmt.Errorf("invalid host ip %q", host.IP)
}
ipNet := &net.IPNet{
IP: ip,
Mask: ipCIDRMask,
}
var hostPub, hostKey []byte
{
var pubkey, privkey [32]byte
if _, err := io.ReadFull(rand.Reader, privkey[:]); err != nil {
return fmt.Errorf("reading random bytes to form private key: %w", err)
}
curve25519.ScalarBaseMult(&pubkey, &privkey)
hostPub, hostKey = pubkey[:], privkey[:]
}
hostCrt := cert.NebulaCertificate{
Details: cert.NebulaCertificateDetails{
Name: host.Name,
Ips: []*net.IPNet{ipNet},
NotBefore: time.Now(),
NotAfter: expireAt,
PublicKey: hostPub,
IsCA: false,
Issuer: issuer,
},
}
if err := hostCrt.CheckRootConstrains(caCrt); err != nil {
return fmt.Errorf("validating certificate constraints: %w", err)
}
if err := hostCrt.Sign(caKey); err != nil {
return fmt.Errorf("signing host cert with ca.key: %w", err)
}
hostKeyPEM := cert.MarshalX25519PrivateKey(hostKey)
hostCrtPEM, err := hostCrt.MarshalToPEM()
if err != nil {
return fmt.Errorf("marshalling host.crt: %w", err)
}
w.WriteFileBytes("nebula/certs/ca.crt", caCrtPEM)
w.WriteFileBytes("nebula/certs/host.key", hostKeyPEM)
w.WriteFileBytes("nebula/certs/host.crt", hostCrtPEM)
return nil
}
// NewForHost generates a new bootstrap file for an arbitrary host, based on the
// given admin file's FS and data in garage.
func NewForHost(env *crypticnet.Env, adminFS fs.FS, name string, into io.Writer) error {
host, ok := env.Hosts[name]
if !ok {
return fmt.Errorf("unknown host %q, make sure host entry has been created", name)
}
client, err := garage.GlobalBucketAPIClient(env)
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
w := tarutil.NewTGZWriter(into)
w.WriteFileBytes("hostname", []byte(name))
if err := writeNewNebulaCert(w, adminFS, host.Nebula); err != nil {
return fmt.Errorf("creating/adding host's nebula certs: %w", err)
}
fsFilesToCopy := []string{
"garage/rpc-secret.txt",
"garage/cryptic-net-global-bucket-key.yml",
}
for _, filePath := range fsFilesToCopy {
if err := copyFSFile(w, adminFS, filePath); err != nil {
return fmt.Errorf("copying %q from bootstrap fs: %w", filePath, err)
}
}
garageDirsToCopy := []string{
"nebula/hosts",
"garage/hosts",
}
for _, dirPath := range garageDirsToCopy {
if err := copyGarageDir(env.Context, client, w, dirPath); err != nil {
return fmt.Errorf("copying %q from garage: %w", dirPath, err)
}
}
return w.Close()
}

View File

@ -0,0 +1,49 @@
package bootstrap
import (
crypticnet "cryptic-net"
"cryptic-net/garage"
"cryptic-net/tarutil"
"fmt"
"io"
)
// NewForThisHost generates a new bootstrap file for the current host, based on
// the existing environment as well as data in garage.
func NewForThisHost(env *crypticnet.Env, into io.Writer) error {
client, err := garage.GlobalBucketAPIClient(env)
if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err)
}
w := tarutil.NewTGZWriter(into)
fsFilesToCopy := []string{
"hostname",
"nebula/certs/ca.crt",
"nebula/certs/host.crt",
"nebula/certs/host.key",
"garage/rpc-secret.txt",
"garage/cryptic-net-global-bucket-key.yml",
}
for _, filePath := range fsFilesToCopy {
if err := copyFSFile(w, env.BootstrapFS, filePath); err != nil {
return fmt.Errorf("copying %q from bootstrap fs: %w", filePath, err)
}
}
garageDirsToCopy := []string{
"nebula/hosts",
"garage/hosts",
}
for _, dirPath := range garageDirsToCopy {
if err := copyGarageDir(env.Context, client, w, dirPath); err != nil {
return fmt.Errorf("copying %q from garage: %w", dirPath, err)
}
}
return w.Close()
}

View File

@ -0,0 +1,74 @@
package bootstrap
import (
"context"
"cryptic-net/garage"
"cryptic-net/tarutil"
"fmt"
"io/fs"
"github.com/minio/minio-go/v7"
)
func copyFSFile(w *tarutil.TGZWriter, srcFS fs.FS, path string) error {
f, err := srcFS.Open(path)
if err != nil {
return fmt.Errorf("opening %q in bootstrap fs: %w", path, err)
}
defer f.Close()
fStat, err := f.Stat()
if err != nil {
return fmt.Errorf("stating %q from bootstrap fs: %w", path, err)
}
w.WriteFile(path, fStat.Size(), f)
return nil
}
func copyGarageDir(
ctx context.Context, client *minio.Client,
w *tarutil.TGZWriter, path string,
) error {
objInfoCh := client.ListObjects(
ctx, garage.GlobalBucket,
minio.ListObjectsOptions{
Prefix: path,
Recursive: true,
},
)
for objInfo := range objInfoCh {
if objInfo.Err != nil {
return fmt.Errorf("listing objects: %w", objInfo.Err)
}
obj, err := client.GetObject(
ctx, garage.GlobalBucket, objInfo.Key, minio.GetObjectOptions{},
)
if err != nil {
return fmt.Errorf(
"retrieving object %q from global bucket: %w",
objInfo.Key, err,
)
}
objStat, err := obj.Stat()
if err != nil {
obj.Close()
return fmt.Errorf(
"stating object %q from global bucket: %w",
objInfo.Key, err,
)
}
w.WriteFile(objInfo.Key, objStat.Size, obj)
obj.Close()
}
return nil
}

View File

@ -14,7 +14,6 @@ import (
crypticnet "cryptic-net" crypticnet "cryptic-net"
"cryptic-net/bootstrap" "cryptic-net/bootstrap"
bootstrap_creator "cryptic-net/bootstrap/creator"
"cryptic-net/yamlutil" "cryptic-net/yamlutil"
"github.com/cryptic-io/pmux/pmuxlib" "github.com/cryptic-io/pmux/pmuxlib"
@ -112,16 +111,21 @@ func reloadBootstrap(env *crypticnet.Env) (bool, error) {
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
if err := bootstrap_creator.NewForThisHost(env, buf); err != nil { if err := bootstrap.NewForThisHost(env, buf); err != nil {
return false, fmt.Errorf("generating new bootstrap from env: %w", err) return false, fmt.Errorf("generating new bootstrap from env: %w", err)
} }
newBootstrap, err := bootstrap.FromReader(bytes.NewReader(buf.Bytes())) newHash, err := bootstrap.GetHashFromReader(bytes.NewReader(buf.Bytes()))
if err != nil { if err != nil {
return false, fmt.Errorf("parsing bootstrap which was just created: %w", err) return false, fmt.Errorf("reading hash from new bootstrap file: %w", err)
} }
if bytes.Equal(newBootstrap.Hash, env.Bootstrap.Hash) { currHash, err := bootstrap.GetHashFromFS(env.BootstrapFS)
if err != nil {
return false, fmt.Errorf("reading hash from existing bootstrap fs: %w", err)
}
if bytes.Equal(newHash, currHash) {
return false, nil return false, nil
} }
@ -132,12 +136,12 @@ func reloadBootstrap(env *crypticnet.Env) (bool, error) {
return true, nil return true, nil
} }
// runs a single pmux process ofor daemon, returning only once the env.Context // runs a single pmux process for daemon, returning only once the env.Context
// has been canceled or bootstrap info has been changed. This will always block // has been canceled or bootstrap info has been changed. This will always block
// until the spawned pmux has returned. // until the spawned pmux has returned.
func runDaemonPmuxOnce(env *crypticnet.Env) error { func runDaemonPmuxOnce(env *crypticnet.Env) error {
thisHost := env.Bootstrap.ThisHost() thisHost := env.ThisHost()
fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP) fmt.Fprintf(os.Stderr, "host name is %q, ip is %q\n", thisHost.Name, thisHost.Nebula.IP)
pmuxProcConfigs := []pmuxlib.ProcessConfig{ pmuxProcConfigs := []pmuxlib.ProcessConfig{
@ -153,7 +157,7 @@ func runDaemonPmuxOnce(env *crypticnet.Env) error {
Cmd: "bash", Cmd: "bash",
Args: []string{ Args: []string{
"wait-for-ip", "wait-for-ip",
thisHost.Nebula.IP, env.ThisHost().Nebula.IP,
"bash", "bash",
"dnsmasq-entrypoint", "dnsmasq-entrypoint",
}, },
@ -166,7 +170,7 @@ func runDaemonPmuxOnce(env *crypticnet.Env) error {
Cmd: "bash", Cmd: "bash",
Args: []string{ Args: []string{
"wait-for-ip", "wait-for-ip",
thisHost.Nebula.IP, env.ThisHost().Nebula.IP,
"cryptic-net-main", "garage-entrypoint", "cryptic-net-main", "garage-entrypoint",
}, },

View File

@ -2,10 +2,31 @@ package entrypoint
import ( import (
"fmt" "fmt"
"io/fs"
"log"
"os" "os"
"strings"
"syscall" "syscall"
crypticnet "cryptic-net"
"cryptic-net/garage"
) )
func getGaragePeer(env *crypticnet.Env) (string, error) {
if allocs := env.ThisDaemon().Storage.Allocations; len(allocs) > 0 {
return garage.GeneratePeerAddr(env.ThisHost().Nebula.IP, allocs[0].RPCPort)
}
bootstrapPeers, err := garage.BootstrapPeerAddrs(env.Hosts)
if err != nil {
return "", err
}
return bootstrapPeers[0], nil
}
var subCmdGarageMC = subCmd{ var subCmdGarageMC = subCmd{
name: "mc", name: "mc",
descr: "Runs the mc (minio-client) binary. The cryptic-net garage can be accessed under the `garage` alias", descr: "Runs the mc (minio-client) binary. The cryptic-net garage can be accessed under the `garage` alias",
@ -30,16 +51,21 @@ var subCmdGarageMC = subCmd{
env := subCmdCtx.env env := subCmdCtx.env
s3APIAddr := env.ChooseGaragePeer().S3APIAddr() apiAddr := garage.APIAddr(env)
if *keyID == "" || *keySecret == "" { if *keyID == "" || *keySecret == "" {
globalBucketCreds, err := garage.GlobalBucketAPICredentials(env)
if err != nil {
return fmt.Errorf("loading global bucket credentials: %w", err)
}
if *keyID == "" { if *keyID == "" {
*keyID = env.Bootstrap.GarageGlobalBucketS3APICredentials.ID *keyID = globalBucketCreds.ID
} }
if *keySecret == "" { if *keySecret == "" {
*keyID = env.Bootstrap.GarageGlobalBucketS3APICredentials.Secret *keySecret = globalBucketCreds.Secret
} }
} }
@ -57,7 +83,7 @@ var subCmdGarageMC = subCmd{
os.Environ(), os.Environ(),
fmt.Sprintf( fmt.Sprintf(
"MC_HOST_garage=http://%s:%s@%s", "MC_HOST_garage=http://%s:%s@%s",
*keyID, *keySecret, s3APIAddr, *keyID, *keySecret, apiAddr,
), ),
// The garage docs say this is necessary, though nothing bad // The garage docs say this is necessary, though nothing bad
@ -85,13 +111,27 @@ var subCmdGarageCLI = subCmd{
env := subCmdCtx.env env := subCmdCtx.env
peerAddr, err := getGaragePeer(env)
if err != nil {
return fmt.Errorf("picking peer to communicate with: %w", err)
}
rpcSecretB, err := fs.ReadFile(env.BootstrapFS, "garage/rpc-secret.txt")
if err != nil {
log.Fatalf("reading garage rpc secret bootstrap fs: %v", err)
}
rpcSecret := strings.TrimSpace(string(rpcSecretB))
var ( var (
binPath = env.BinPath("garage") binPath = env.BinPath("garage")
args = append([]string{"garage"}, subCmdCtx.args...) args = append([]string{"garage"}, subCmdCtx.args...)
cliEnv = append( cliEnv = append(
os.Environ(), os.Environ(),
"GARAGE_RPC_HOST="+env.ChooseGaragePeer().RPCAddr(), "GARAGE_RPC_HOST="+peerAddr,
"GARAGE_RPC_SECRET="+env.Bootstrap.GarageRPCSecret, "GARAGE_RPC_SECRET="+rpcSecret,
) )
) )

View File

@ -2,8 +2,8 @@ package entrypoint
import ( import (
"bytes" "bytes"
crypticnet "cryptic-net"
"cryptic-net/bootstrap" "cryptic-net/bootstrap"
bootstrap_creator "cryptic-net/bootstrap/creator"
"cryptic-net/garage" "cryptic-net/garage"
"cryptic-net/tarutil" "cryptic-net/tarutil"
"errors" "errors"
@ -73,12 +73,12 @@ var subCmdHostsAdd = subCmd{
env := subCmdCtx.env env := subCmdCtx.env
client, err := env.GlobalBucketS3APIClient() client, err := garage.GlobalBucketAPIClient(env)
if err != nil { if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err) return fmt.Errorf("creating client for global bucket: %w", err)
} }
nebulaHost := bootstrap.NebulaHost{ nebulaHost := crypticnet.NebulaHost{
Name: *name, Name: *name,
IP: *ip, IP: *ip,
} }
@ -113,7 +113,7 @@ var subCmdHostsList = subCmd{
env := subCmdCtx.env env := subCmdCtx.env
client, err := env.GlobalBucketS3APIClient() client, err := garage.GlobalBucketAPIClient(env)
if err != nil { if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err) return fmt.Errorf("creating client for global bucket: %w", err)
} }
@ -147,7 +147,7 @@ var subCmdHostsList = subCmd{
return fmt.Errorf("retrieving object %q from global bucket: %w", objInfo.Key, err) return fmt.Errorf("retrieving object %q from global bucket: %w", objInfo.Key, err)
} }
var nebulaHost bootstrap.NebulaHost var nebulaHost crypticnet.NebulaHost
err = yaml.NewDecoder(obj).Decode(&nebulaHost) err = yaml.NewDecoder(obj).Decode(&nebulaHost)
obj.Close() obj.Close()
@ -191,7 +191,7 @@ var subCmdHostsDelete = subCmd{
filePath := nebulaHostPath(*name) filePath := nebulaHostPath(*name)
client, err := env.GlobalBucketS3APIClient() client, err := garage.GlobalBucketAPIClient(env)
if err != nil { if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err) return fmt.Errorf("creating client for global bucket: %w", err)
} }
@ -263,7 +263,7 @@ var subCmdHostsMakeBootstrap = subCmd{
return fmt.Errorf("reading admin.tgz with --admin-path of %q: %w", *adminPath, err) return fmt.Errorf("reading admin.tgz with --admin-path of %q: %w", *adminPath, err)
} }
return bootstrap_creator.NewForHost(subCmdCtx.env, adminFS, *name, os.Stdout) return bootstrap.NewForHost(subCmdCtx.env, adminFS, *name, os.Stdout)
}, },
} }

View File

@ -2,11 +2,13 @@ package garage_entrypoint
import ( import (
"fmt" "fmt"
"io/fs"
"log" "log"
"net" "net"
"os" "os"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings"
"time" "time"
crypticnet "cryptic-net" crypticnet "cryptic-net"
@ -17,23 +19,24 @@ import (
func writeChildConf( func writeChildConf(
env *crypticnet.Env, env *crypticnet.Env,
bootstrapPeers []string,
alloc crypticnet.DaemonYmlStorageAllocation, alloc crypticnet.DaemonYmlStorageAllocation,
rpcSecret string,
) (string, error) { ) (string, error) {
if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil { if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err) return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
} }
thisHost := env.Bootstrap.ThisHost() pubKey, privKey, err := garage.GeneratePeerKey(env.ThisHost().Nebula.IP, alloc.RPCPort)
peer := garage.Peer{ if err != nil {
IP: thisHost.Nebula.IP, return "", fmt.Errorf(
RPCPort: alloc.RPCPort, "generating node key with input %q,%d: %w",
S3APIPort: alloc.S3APIPort, env.ThisHost().Nebula.IP, alloc.RPCPort, err,
)
} }
pubKey, privKey := peer.RPCPeerKey()
nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key") nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub") nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
@ -48,17 +51,17 @@ func writeChildConf(
env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort), env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
) )
err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{ err = garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
MetaPath: alloc.MetaPath, MetaPath: alloc.MetaPath,
DataPath: alloc.DataPath, DataPath: alloc.DataPath,
RPCSecret: env.Bootstrap.GarageRPCSecret, RPCSecret: rpcSecret,
RPCAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)), RPCAddr: net.JoinHostPort(env.ThisHost().Nebula.IP, strconv.Itoa(alloc.RPCPort)),
APIAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)), APIAddr: net.JoinHostPort(env.ThisHost().Nebula.IP, strconv.Itoa(alloc.APIPort)),
WebAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.WebPort)), WebAddr: net.JoinHostPort(env.ThisHost().Nebula.IP, strconv.Itoa(alloc.WebPort)),
BootstrapPeers: env.Bootstrap.GarageRPCPeerAddrs(), BootstrapPeers: bootstrapPeers,
}) })
if err != nil { if err != nil {
@ -70,15 +73,13 @@ func writeChildConf(
func waitForArgs(env *crypticnet.Env, bin string, binArgs ...string) []string { func waitForArgs(env *crypticnet.Env, bin string, binArgs ...string) []string {
thisHost := env.Bootstrap.ThisHost()
var args []string var args []string
for _, alloc := range env.ThisDaemon().Storage.Allocations { for _, alloc := range env.ThisDaemon().Storage.Allocations {
args = append( args = append(
args, args,
"wait-for", "wait-for",
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)), net.JoinHostPort(env.ThisHost().Nebula.IP, strconv.Itoa(alloc.RPCPort)),
"--", "--",
) )
} }
@ -97,11 +98,25 @@ func Main() {
log.Fatalf("reading envvars: %v", err) log.Fatalf("reading envvars: %v", err)
} }
bootstrapPeers, err := garage.BootstrapPeerAddrs(env.Hosts)
if err != nil {
log.Fatalf("generating set of bootstrap peers: %v", err)
}
rpcSecretB, err := fs.ReadFile(env.BootstrapFS, "garage/rpc-secret.txt")
if err != nil {
log.Fatalf("reading garage rpc secret bootstrap fs: %v", err)
}
rpcSecret := strings.TrimSpace(string(rpcSecretB))
var pmuxProcConfigs []pmuxlib.ProcessConfig var pmuxProcConfigs []pmuxlib.ProcessConfig
for _, alloc := range env.ThisDaemon().Storage.Allocations { for _, alloc := range env.ThisDaemon().Storage.Allocations {
childConfPath, err := writeChildConf(env, alloc) childConfPath, err := writeChildConf(env, bootstrapPeers, alloc, rpcSecret)
if err != nil { if err != nil {
log.Fatalf("writing child config file for alloc %+v: %v", alloc, err) log.Fatalf("writing child config file for alloc %+v: %v", alloc, err)

View File

@ -148,30 +148,29 @@ func readCurrNodes(r io.Reader) (clusterNodes, int, error) {
return currNodes, version, nil return currNodes, version, nil
} }
func readExpNodes(env *crypticnet.Env) clusterNodes { func readExpNodes(env *crypticnet.Env) (clusterNodes, error) {
thisHost := env.Bootstrap.ThisHost()
var expNodes clusterNodes var expNodes clusterNodes
for _, alloc := range env.ThisDaemon().Storage.Allocations { for _, alloc := range env.ThisDaemon().Storage.Allocations {
peer := garage.Peer{ id, err := garage.GeneratePeerID(env.ThisHost().Nebula.IP, alloc.RPCPort)
IP: thisHost.Nebula.IP,
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
}
id := peer.RPCPeerID() if err != nil {
return nil, fmt.Errorf(
"generating peer id for ip:%q port:%d: %w",
env.ThisHost().Nebula.IP, alloc.RPCPort, err,
)
}
expNodes = append(expNodes, clusterNode{ expNodes = append(expNodes, clusterNode{
ID: id, ID: id,
Zone: env.Bootstrap.HostName, Zone: env.ThisHost().Name,
Capacity: alloc.Capacity / 100, Capacity: alloc.Capacity / 100,
}) })
} }
return expNodes return expNodes, nil
} }
// NOTE: The id formatting for currNodes and expNodes is different; expNodes has // NOTE: The id formatting for currNodes and expNodes is different; expNodes has
@ -233,14 +232,18 @@ func Main() {
for _, node := range currNodes { for _, node := range currNodes {
if env.Bootstrap.HostName != node.Zone { if env.ThisHost().Name != node.Zone {
continue continue
} }
thisCurrNodes = append(thisCurrNodes, node) thisCurrNodes = append(thisCurrNodes, node)
} }
expNodes := readExpNodes(env) expNodes, err := readExpNodes(env)
if err != nil {
panic(fmt.Errorf("reading expected layout from environment: %w", err))
}
lines := diff(thisCurrNodes, expNodes) lines := diff(thisCurrNodes, expNodes)

View File

@ -42,12 +42,11 @@ func Main() {
panic("The arguments -ip, -port, and -danger are required") panic("The arguments -ip, -port, and -danger are required")
} }
peer := garage.Peer{ pubKey, privKey, err := garage.GeneratePeerKey(*ip, *port)
IP: *ip,
RPCPort: *port,
}
pubKey, privKey := peer.RPCPeerKey() if err != nil {
panic(fmt.Errorf("GeneratePeerKey returned: %w", err))
}
fmt.Fprintln(os.Stdout, hex.EncodeToString(pubKey)) fmt.Fprintln(os.Stdout, hex.EncodeToString(pubKey))

View File

@ -3,7 +3,6 @@ package garage_update_global_bucket
import ( import (
"bytes" "bytes"
crypticnet "cryptic-net" crypticnet "cryptic-net"
"cryptic-net/bootstrap"
"cryptic-net/garage" "cryptic-net/garage"
"fmt" "fmt"
"log" "log"
@ -17,15 +16,12 @@ func updateGlobalBucket(env *crypticnet.Env) error {
ctx := env.Context ctx := env.Context
client, err := env.GlobalBucketS3APIClient() client, err := garage.GlobalBucketAPIClient(env)
if err != nil { if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err) return fmt.Errorf("creating client for global bucket: %w", err)
} }
filePath := filepath.Join( filePath := filepath.Join("garage/hosts", env.ThisHost().Name+".yml")
"garage/hosts",
env.Bootstrap.HostName+".yml",
)
daemon := env.ThisDaemon() daemon := env.ThisDaemon()
@ -45,13 +41,13 @@ func updateGlobalBucket(env *crypticnet.Env) error {
return nil return nil
} }
var garageHost bootstrap.GarageHost var garageHost crypticnet.GarageHost
for _, alloc := range daemon.Storage.Allocations { for _, alloc := range daemon.Storage.Allocations {
garageHostInstance := bootstrap.GarageHostInstance{ garageHostInstance := crypticnet.GarageHostInstance{
APIPort: alloc.APIPort,
RPCPort: alloc.RPCPort, RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
WebPort: alloc.WebPort, WebPort: alloc.WebPort,
} }

View File

@ -2,6 +2,8 @@ package nebula_entrypoint
import ( import (
"cryptic-net/yamlutil" "cryptic-net/yamlutil"
"fmt"
"io/fs"
"log" "log"
"net" "net"
"path/filepath" "path/filepath"
@ -25,7 +27,7 @@ func Main() {
staticHostMap = map[string][]string{} staticHostMap = map[string][]string{}
) )
for _, host := range env.Bootstrap.Hosts { for _, host := range env.Hosts {
if host.Nebula.PublicAddr == "" { if host.Nebula.PublicAddr == "" {
continue continue
@ -35,11 +37,26 @@ func Main() {
staticHostMap[host.Nebula.IP] = []string{host.Nebula.PublicAddr} staticHostMap[host.Nebula.IP] = []string{host.Nebula.PublicAddr}
} }
readCertFile := func(name string) string {
if err != nil {
return ""
}
path := filepath.Join("nebula", "certs", name)
var b []byte
if b, err = fs.ReadFile(env.BootstrapFS, path); err != nil {
err = fmt.Errorf("reading %q from bootstrap fs: %w", path, err)
}
return string(b)
}
config := map[string]interface{}{ config := map[string]interface{}{
"pki": map[string]string{ "pki": map[string]string{
"ca": env.Bootstrap.NebulaCertsCACert, "ca": readCertFile("ca.crt"),
"cert": env.Bootstrap.NebulaCertsHostCert, "cert": readCertFile("host.crt"),
"key": env.Bootstrap.NebulaCertsHostKey, "key": readCertFile("host.key"),
}, },
"static_host_map": staticHostMap, "static_host_map": staticHostMap,
"punchy": map[string]bool{ "punchy": map[string]bool{
@ -93,7 +110,7 @@ func Main() {
firewallInbound = append( firewallInbound = append(
firewallInbound, firewallInbound,
crypticnet.ConfigFirewallRule{ crypticnet.ConfigFirewallRule{
Port: strconv.Itoa(alloc.S3APIPort), Port: strconv.Itoa(alloc.APIPort),
Proto: "tcp", Proto: "tcp",
Host: "any", Host: "any",
}, },

View File

@ -16,14 +16,14 @@ func updateGlobalBucket(env *crypticnet.Env) error {
ctx := env.Context ctx := env.Context
client, err := env.GlobalBucketS3APIClient() client, err := garage.GlobalBucketAPIClient(env)
if err != nil { if err != nil {
return fmt.Errorf("creating client for global bucket: %w", err) return fmt.Errorf("creating client for global bucket: %w", err)
} }
daemon := env.ThisDaemon() daemon := env.ThisDaemon()
host := env.Bootstrap.ThisHost() host := env.ThisHost()
host.Nebula.Name = host.Name host.Nebula.Name = host.Name
host.Nebula.PublicAddr = daemon.VPN.PublicAddr host.Nebula.PublicAddr = daemon.VPN.PublicAddr

View File

@ -31,7 +31,7 @@ type DaemonYmlStorageAllocation struct {
DataPath string `yaml:"data_path"` DataPath string `yaml:"data_path"`
MetaPath string `yaml:"meta_path"` MetaPath string `yaml:"meta_path"`
Capacity int `yaml:"capacity"` Capacity int `yaml:"capacity"`
S3APIPort int `yaml:"api_port"` // TODO fix field name here APIPort int `yaml:"api_port"`
RPCPort int `yaml:"rpc_port"` RPCPort int `yaml:"rpc_port"`
WebPort int `yaml:"web_port"` WebPort int `yaml:"web_port"`
} }

View File

@ -2,7 +2,7 @@ package crypticnet
import ( import (
"context" "context"
"cryptic-net/bootstrap" "cryptic-net/tarutil"
"cryptic-net/yamlutil" "cryptic-net/yamlutil"
"errors" "errors"
"fmt" "fmt"
@ -37,7 +37,9 @@ type Env struct {
// If NewEnv is called with bootstrapOptional, and a bootstrap file is not // If NewEnv is called with bootstrapOptional, and a bootstrap file is not
// found, then these fields will not be set. // found, then these fields will not be set.
BootstrapPath string BootstrapPath string
Bootstrap bootstrap.Bootstrap BootstrapFS fs.FS
Hosts map[string]Host
HostName string
thisDaemon DaemonYml thisDaemon DaemonYml
thisDaemonOnce sync.Once thisDaemonOnce sync.Once
@ -116,13 +118,32 @@ func (e *Env) DataDirBootstrapPath() string {
// and all derived fields based on that. // and all derived fields based on that.
func (e *Env) LoadBootstrap(path string) error { func (e *Env) LoadBootstrap(path string) error {
var err error var (
err error
if e.Bootstrap, err = bootstrap.FromFile(path); err != nil { // load all values into temp variables before setting the fields on Env,
return fmt.Errorf("parsing bootstrap.tgz at %q: %w", path, err) // so we don't leave it in an inconsistent state.
bootstrapFS fs.FS
hosts map[string]Host
hostNameB []byte
)
if bootstrapFS, err = tarutil.FSFromTGZFile(path); err != nil {
return fmt.Errorf("reading bootstrap file at %q: %w", e.BootstrapPath, err)
}
if hosts, err = LoadHosts(bootstrapFS); err != nil {
return fmt.Errorf("loading hosts info from bootstrap fs: %w", err)
}
if hostNameB, err = fs.ReadFile(bootstrapFS, "hostname"); err != nil {
return fmt.Errorf("loading hostname from bootstrap fs: %w", err)
} }
e.BootstrapPath = path e.BootstrapPath = path
e.BootstrapFS = bootstrapFS
e.Hosts = hosts
e.HostName = string(hostNameB)
return nil return nil
} }
@ -198,6 +219,11 @@ func (e *Env) init(bootstrapOptional bool) error {
return nil return nil
} }
// ThisHost is a shortcut for returning env.Hosts[env.HostName].
func (e *Env) ThisHost() Host {
return e.Hosts[e.HostName]
}
// ToMap returns the Env as a map of key/value strings. If this map is set into // ToMap returns the Env as a map of key/value strings. If this map is set into
// a process's environment, then that process can read it back using ReadEnv. // a process's environment, then that process can read it back using ReadEnv.
func (e *Env) ToMap() map[string]string { func (e *Env) ToMap() map[string]string {

View File

@ -1,42 +0,0 @@
package crypticnet
import (
"cryptic-net/garage"
"fmt"
)
// ChooseGaragePeer returns a Peer for a garage instance from the network. It
// will prefer a garage instance on this particular host, if there is one, but
// will otherwise return a random endpoint.
func (env *Env) ChooseGaragePeer() garage.Peer {
if allocs := env.ThisDaemon().Storage.Allocations; len(allocs) > 0 {
return garage.Peer{
IP: env.Bootstrap.ThisHost().Nebula.IP,
RPCPort: allocs[0].RPCPort,
S3APIPort: allocs[0].S3APIPort,
}
}
for _, peer := range env.Bootstrap.GaragePeers() {
return peer
}
panic("no garage instances configured")
}
// GlobalBucketS3APIClient returns an S3 client pre-configured with access to
// the global bucket.
func (env *Env) GlobalBucketS3APIClient() (garage.S3APIClient, error) {
addr := env.ChooseGaragePeer().S3APIAddr()
creds := env.Bootstrap.GarageGlobalBucketS3APICredentials
client, err := garage.NewS3APIClient(addr, creds)
if err != nil {
return nil, fmt.Errorf("connecting to garage S3 API At %q: %w", addr, err)
}
return client, err
}

View File

@ -1,7 +1,12 @@
package garage package garage
import ( import (
crypticnet "cryptic-net"
"cryptic-net/yamlutil"
"errors" "errors"
"fmt"
"net"
"strconv"
"github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/credentials"
@ -14,21 +19,74 @@ func IsKeyNotFound(err error) bool {
return errors.As(err, &mErr) && mErr.Code == "NoSuchKey" return errors.As(err, &mErr) && mErr.Code == "NoSuchKey"
} }
// S3APIClient is a client used to interact with garage's S3 API. // APICredentials describe data fields necessary for authenticating with a
type S3APIClient = *minio.Client // garage api endpoint.
type APICredentials struct {
// S3APICredentials describe data fields necessary for authenticating with a
// garage S3 API endpoint.
type S3APICredentials struct {
ID string `yaml:"id"` ID string `yaml:"id"`
Secret string `yaml:"secret"` Secret string `yaml:"secret"`
} }
// NewS3APIClient returns a minio client configured to use the given garage S3 API // GlobalBucketAPICredentials returns APICredentials for the global bucket.
func GlobalBucketAPICredentials(env *crypticnet.Env) (APICredentials, error) {
const path = "garage/cryptic-net-global-bucket-key.yml"
var creds APICredentials
if err := yamlutil.LoadYamlFSFile(&creds, env.BootstrapFS, path); err != nil {
return APICredentials{}, fmt.Errorf("loading %q from bootstrap fs: %w", path, err)
}
return creds, nil
}
// APIAddr returns the network address of a garage api endpoint in the network.
// It will prefer an endpoint on this particular host, if there is one, but will
// otherwise return a random endpoint.
func APIAddr(env *crypticnet.Env) string {
if allocs := env.ThisDaemon().Storage.Allocations; len(allocs) > 0 {
return net.JoinHostPort(
env.ThisHost().Nebula.IP,
strconv.Itoa(allocs[0].APIPort),
)
}
for _, host := range env.Hosts {
if host.Garage == nil || len(host.Garage.Instances) == 0 {
continue
}
return net.JoinHostPort(
host.Nebula.IP,
strconv.Itoa(host.Garage.Instances[0].APIPort),
)
}
panic("no garage instances configured")
}
// APIClient returns a minio client configured to use the given garage API
// endpoint. // endpoint.
func NewS3APIClient(addr string, creds S3APICredentials) (S3APIClient, error) { func APIClient(addr string, creds APICredentials) (*minio.Client, error) {
return minio.New(addr, &minio.Options{ return minio.New(addr, &minio.Options{
Creds: credentials.NewStaticV4(creds.ID, creds.Secret, ""), Creds: credentials.NewStaticV4(creds.ID, creds.Secret, ""),
Region: Region, Region: Region,
}) })
} }
// GlobalBucketAPIClient returns a minio client pre-configured with access to
// the global bucket.
func GlobalBucketAPIClient(env *crypticnet.Env) (*minio.Client, error) {
creds, err := GlobalBucketAPICredentials(env)
if err != nil {
return nil, fmt.Errorf("loading global bucket credentials: %w", err)
}
addr := APIAddr(env)
return APIClient(addr, creds)
}

View File

@ -1,13 +0,0 @@
// Package garage contains helper functions and types which are useful for
// setting up garage configs, processes, and deployments.
package garage
const (
// Region is the region which garage is configured with.
Region = "garage"
// GlobalBucket is the name of the global garage bucket which is
// accessible to all hosts in the network.
GlobalBucket = "cryptic-net-global"
)

View File

@ -0,0 +1,100 @@
// Package garage contains helper functions and types which are useful for
// setting up garage configs, processes, and deployments.
package garage
import (
crypticnet "cryptic-net"
"crypto/ed25519"
"encoding/hex"
"fmt"
"net"
"strconv"
)
const (
// Region is the region which garage is configured with.
Region = "garage"
// GlobalBucket is the name of the global garage bucket which is
// accessible to all hosts in the network.
GlobalBucket = "cryptic-net-global"
)
// GeneratePeerKey deterministically generates a public/private keys which can
// be used as a garage node key.
//
// DANGER: This function will deterministically produce public/private keys
// given some arbitrary input. This is NEVER what you want. It's only being used
// in cryptic-net for a very specific purpose for which I think it's ok and is
// very necessary, and people are probably _still_ going to yell at me.
//
func GeneratePeerKey(ip string, port int) (pubKey, privKey []byte, err error) {
input := []byte(net.JoinHostPort(ip, strconv.Itoa(port)))
// Append the length of the input to the input, so that the input "foo"
// doesn't generate the same key as the input "foofoo".
input = strconv.AppendInt(input, int64(len(input)), 10)
return ed25519.GenerateKey(NewInfiniteReader(input))
}
// GeneratePeerID generates the peer id for the given peer.
//
// DANGER: See warning on GenerateNodeKey.
func GeneratePeerID(ip string, port int) (string, error) {
peerNodeKeyPub, _, err := GeneratePeerKey(ip, port)
if err != nil {
return "", err
}
return hex.EncodeToString(peerNodeKeyPub), nil
}
// GeneratePeerAddr generates the peer address (e.g. "id@ip:port") for the
// given peer.
//
// DANGER: See warning on GenerateNodeKey.
func GeneratePeerAddr(ip string, port int) (string, error) {
id, err := GeneratePeerID(ip, port)
if err != nil {
return "", fmt.Errorf("generating peer id: %w", err)
}
return fmt.Sprintf("%s@%s", id, net.JoinHostPort(ip, strconv.Itoa(port))), nil
}
// BootstrapPeerAddrs generates the list of bootstrap peer strings based on the
// bootstrap hosts.
func BootstrapPeerAddrs(hosts map[string]crypticnet.Host) ([]string, error) {
var peers []string
for _, host := range hosts {
if host.Garage == nil {
continue
}
for _, instance := range host.Garage.Instances {
peer, err := GeneratePeerAddr(host.Nebula.IP, instance.RPCPort)
if err != nil {
return nil, fmt.Errorf(
"generating peer address with input %q,%d: %w",
host.Nebula.IP, instance.RPCPort, err,
)
}
peers = append(peers, peer)
}
}
return peers, nil
}

View File

@ -1,66 +0,0 @@
package garage
import (
"crypto/ed25519"
"encoding/hex"
"fmt"
"net"
"strconv"
)
// Peer describes all information necessary to connect to a given garage node.
type Peer struct {
IP string
RPCPort int
S3APIPort int
}
// RPCPeerKey deterministically generates a public/private keys which can
// be used as a garage node key.
//
// DANGER: This function will deterministically produce public/private keys
// given some arbitrary input. This is NEVER what you want. It's only being used
// in cryptic-net for a very specific purpose for which I think it's ok and is
// very necessary, and people are probably _still_ going to yell at me.
//
func (p Peer) RPCPeerKey() (pubKey, privKey []byte) {
input := []byte(net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort)))
// Append the length of the input to the input, so that the input "foo"
// doesn't generate the same key as the input "foofoo".
input = strconv.AppendInt(input, int64(len(input)), 10)
pubKey, privKey, err := ed25519.GenerateKey(NewInfiniteReader(input))
if err != nil {
panic(err)
}
return pubKey, privKey
}
// RPCPeerID returns the peer ID of the garage node for use in communicating
// over RPC.
//
// DANGER: See warning on RPCPeerKey.
func (p Peer) RPCPeerID() string {
pubKey, _ := p.RPCPeerKey()
return hex.EncodeToString(pubKey)
}
// RPCAddr returns the address of the peer's RPC port.
func (p Peer) RPCAddr() string {
return net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort))
}
// RPCPeerAddr returns the full peer address (e.g. "id@ip:port") of the garage
// node for use in communicating over RPC.
//
// DANGER: See warning on RPCPeerKey.
func (p Peer) RPCPeerAddr() string {
return fmt.Sprintf("%s@%s", p.RPCPeerID(), p.RPCAddr())
}
// S3APIAddr returns the address of the peer's S3 API port.
func (p Peer) S3APIAddr() string {
return net.JoinHostPort(p.IP, strconv.Itoa(p.S3APIPort))
}

View File

@ -1,4 +1,4 @@
package bootstrap package crypticnet
import ( import (
"errors" "errors"
@ -19,8 +19,8 @@ type NebulaHost struct {
// GarageHostInstance describes a single garage instance running on a host. // GarageHostInstance describes a single garage instance running on a host.
type GarageHostInstance struct { type GarageHostInstance struct {
APIPort int `yaml:"api_port"`
RPCPort int `yaml:"rpc_port"` RPCPort int `yaml:"rpc_port"`
S3APIPort int `yaml:"s3_api_port"`
WebPort int `yaml:"web_port"` WebPort int `yaml:"web_port"`
} }
@ -37,7 +37,8 @@ type Host struct {
Garage *GarageHost Garage *GarageHost
} }
func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) { // LostHosts returns a mapping of hostnames to Host objects for each host.
func LoadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
hosts := map[string]Host{} hosts := map[string]Host{}
@ -51,11 +52,9 @@ func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
} }
{ {
globPath := filepath.Join(NebulaHostsDirPath, "*.yml") nebulaHostFiles, err := fs.Glob(bootstrapFS, "nebula/hosts/*.yml")
nebulaHostFiles, err := fs.Glob(bootstrapFS, globPath)
if err != nil { if err != nil {
return nil, fmt.Errorf("listing nebula host files at %q in fs: %w", globPath, err) return nil, fmt.Errorf("listing nebula host files: %w", err)
} }
for _, nebulaHostPath := range nebulaHostFiles { for _, nebulaHostPath := range nebulaHostFiles {
@ -77,7 +76,7 @@ func loadHosts(bootstrapFS fs.FS) (map[string]Host, error) {
for hostName, host := range hosts { for hostName, host := range hosts {
garageHostPath := filepath.Join(GarageHostsDirPath, hostName+".yml") garageHostPath := filepath.Join("garage/hosts", hostName+".yml")
var garageHost GarageHost var garageHost GarageHost
if err := readAsYaml(&garageHost, garageHostPath); errors.Is(err, fs.ErrNotExist) { if err := readAsYaml(&garageHost, garageHostPath); errors.Is(err, fs.ErrNotExist) {

View File

@ -1,172 +0,0 @@
// Package nebula contains helper functions and types which are useful for
// setting up nebula configs, processes, and deployments.
package nebula
import (
"cryptic-net/bootstrap"
"crypto/ed25519"
"crypto/rand"
"fmt"
"io"
"io/fs"
"net"
"time"
"github.com/slackhq/nebula/cert"
"golang.org/x/crypto/curve25519"
)
// TODO this should one day not be hardcoded
var ipCIDRMask = func() net.IPMask {
_, ipNet, err := net.ParseCIDR("10.10.0.0/16")
if err != nil {
panic(err)
}
return ipNet.Mask
}()
// HostCert contains the certificate and private key files which will need to
// be present on a particular host. Each file is PEM encoded.
type HostCert struct {
CACert []byte
HostKey []byte
HostCert []byte
}
// CACert contains the certificate and private files which can be used to create
// HostCerts. Each file is PEM encoded.
type CACert struct {
CACert []byte
CAKey []byte
}
// NewHostCert generates a new key/cert for a nebula host using the CA key
// which will be found in the adminFS.
func NewHostCert(
adminFS fs.FS, host bootstrap.NebulaHost,
) (
HostCert, error,
) {
// The logic here is largely based on
// https://github.com/slackhq/nebula/blob/v1.4.0/cmd/nebula-cert/sign.go
caKeyPEM, err := fs.ReadFile(adminFS, "nebula/certs/ca.key")
if err != nil {
return HostCert{}, fmt.Errorf("reading ca.key from admin fs: %w", err)
}
caKey, _, err := cert.UnmarshalEd25519PrivateKey(caKeyPEM)
if err != nil {
return HostCert{}, fmt.Errorf("unmarshaling ca.key: %w", err)
}
caCrtPEM, err := fs.ReadFile(adminFS, "nebula/certs/ca.crt")
if err != nil {
return HostCert{}, fmt.Errorf("reading ca.crt from admin fs: %w", err)
}
caCrt, _, err := cert.UnmarshalNebulaCertificateFromPEM(caCrtPEM)
if err != nil {
return HostCert{}, fmt.Errorf("unmarshaling ca.crt: %w", err)
}
issuer, err := caCrt.Sha256Sum()
if err != nil {
return HostCert{}, fmt.Errorf("getting ca.crt issuer: %w", err)
}
expireAt := caCrt.Details.NotAfter.Add(-1 * time.Second)
ip := net.ParseIP(host.IP)
if ip == nil {
return HostCert{}, fmt.Errorf("invalid host ip %q", host.IP)
}
ipNet := &net.IPNet{
IP: ip,
Mask: ipCIDRMask,
}
var hostPub, hostKey []byte
{
var pubkey, privkey [32]byte
if _, err := io.ReadFull(rand.Reader, privkey[:]); err != nil {
return HostCert{}, fmt.Errorf("reading random bytes to form private key: %w", err)
}
curve25519.ScalarBaseMult(&pubkey, &privkey)
hostPub, hostKey = pubkey[:], privkey[:]
}
hostCrt := cert.NebulaCertificate{
Details: cert.NebulaCertificateDetails{
Name: host.Name,
Ips: []*net.IPNet{ipNet},
NotBefore: time.Now(),
NotAfter: expireAt,
PublicKey: hostPub,
IsCA: false,
Issuer: issuer,
},
}
if err := hostCrt.CheckRootConstrains(caCrt); err != nil {
return HostCert{}, fmt.Errorf("validating certificate constraints: %w", err)
}
if err := hostCrt.Sign(caKey); err != nil {
return HostCert{}, fmt.Errorf("signing host cert with ca.key: %w", err)
}
hostKeyPEM := cert.MarshalX25519PrivateKey(hostKey)
hostCrtPEM, err := hostCrt.MarshalToPEM()
if err != nil {
return HostCert{}, fmt.Errorf("marshalling host.crt: %w", err)
}
return HostCert{
CACert: caCrtPEM,
HostKey: hostKeyPEM,
HostCert: hostCrtPEM,
}, nil
}
// NewCACert generates a CACert. The domain should be the network's root domain,
// and is included in the signing certificate's Name field.
func NewCACert(domain string) (CACert, error) {
pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
panic(fmt.Errorf("generating ed25519 key: %w", err))
}
now := time.Now()
expireAt := now.Add(2 * 365 * 24 * time.Hour)
caCrt := cert.NebulaCertificate{
Details: cert.NebulaCertificateDetails{
Name: fmt.Sprintf("%s cryptic-net root cert", domain),
NotBefore: now,
NotAfter: expireAt,
PublicKey: pubKey,
IsCA: true,
},
}
if err := caCrt.Sign(privKey); err != nil {
return CACert{}, fmt.Errorf("signing caCrt: %w", err)
}
caKeyPEM := cert.MarshalEd25519PrivateKey(privKey)
caCrtPem, err := caCrt.MarshalToPEM()
if err != nil {
return CACert{}, fmt.Errorf("marshaling caCrt: %w", err)
}
return CACert{
CACert: caCrtPem,
CAKey: caKeyPEM,
}, nil
}

View File

@ -7,11 +7,12 @@ import (
"fmt" "fmt"
"io" "io"
"io/fs" "io/fs"
"os"
"github.com/nlepage/go-tarfs" "github.com/nlepage/go-tarfs"
) )
// FSFromReader returns a FS instance which will read the contents of a tgz // FSFromTGZFile returns a FS instance which will read the contents of a tgz
// file from the given Reader. // file from the given Reader.
func FSFromReader(r io.Reader) (fs.FS, error) { func FSFromReader(r io.Reader) (fs.FS, error) {
gf, err := gzip.NewReader(r) gf, err := gzip.NewReader(r)
@ -22,3 +23,15 @@ func FSFromReader(r io.Reader) (fs.FS, error) {
return tarfs.New(gf) return tarfs.New(gf)
} }
// FSFromTGZFile returns a FS instance which will read the contents of a tgz
// file.
func FSFromTGZFile(path string) (fs.FS, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("opening file: %w", err)
}
defer f.Close()
return FSFromReader(f)
}

View File

@ -7,7 +7,6 @@ import (
"crypto/sha512" "crypto/sha512"
"fmt" "fmt"
"io" "io"
"io/fs"
"path/filepath" "path/filepath"
"sort" "sort"
"strings" "strings"
@ -150,22 +149,3 @@ func (w *TGZWriter) WriteFileBytes(path string, body []byte) {
bodyR := bytes.NewReader(body) bodyR := bytes.NewReader(body)
w.WriteFile(path, bodyR.Size(), bodyR) w.WriteFile(path, bodyR.Size(), bodyR)
} }
// CopyFileFromFS copies the file at the given path from srcFS into the same
// path in the TGZWriter.
func (w *TGZWriter) CopyFileFromFS(path string, srcFS fs.FS) error {
f, err := srcFS.Open(path)
if err != nil {
return fmt.Errorf("opening: %w", err)
}
defer f.Close()
fStat, err := f.Stat()
if err != nil {
return fmt.Errorf("stating: %w", err)
}
w.WriteFile(path, fStat.Size(), f)
return nil
}