AppDir/etc/daemon.yml

@@ -69,7 +69,7 @@ storage:
   # the meta directories can be anywhere (ideally on an SSD).
   #
   # Capacity declares how many gigabytes can be stored in each allocation, and
-  # is required.
+  # is required. It must be a multiple of 100.
   #
   # The ports are all _optional_, and will be automatically assigned if they are
   # not specified. If ports any ports are specified then all should be

default.nix

@@ -69,7 +69,7 @@ in rec {
       '';
     };
 
-    vendorHash = "sha256-33gwBj+6x9I/yz0Qf4G8YXRgC/HfwHCedqzrCE4FHHk=";
+    vendorHash = "sha256-P1TXG0fG8/6n37LmM5ApYctqoZzJFlvFAO2Zl85SVvk=";
 
     subPackages = [
       "./cmd/entrypoint"
@@ -90,7 +90,7 @@ in rec {
 
     inherit buildSystem;
     hostSystem = "${hostPlatform.cpu.name}-unknown-${hostPlatform.kernel.name}-musl";
-    pkgsSrc = pkgsNix.src;
+    #pkgsSrc = pkgsNix.src;
 
   };
 

docs/operator/contributing-storage.md

@@ -35,7 +35,7 @@ storage:
       meta_path: /mnt/drive1/isle/meta
       capacity: 1200
 
-    # 100 GB are being shared from drive2
+    # 100 GB (the minimum) are being shared from drive2
     - data_path: /mnt/drive2/isle/data
       meta_path: /mnt/drive2/isle/meta
       capacity: 100

go/bootstrap/bootstrap.go

@@ -30,13 +30,10 @@ func AppDirPath(appDirPath string) string {
 
 // Garage contains parameters needed to connect to and use the garage cluster.
 type Garage struct {
-	// TODO this should be part of some new configuration section related to
-	// secrets which may or may not be granted to this host
-	RPCSecret string
-
-	AdminToken string
-
-	// TODO this should be part of admin.CreationParams
+	// TODO RPCSecret and GlobalBucketS3APICredentials are duplicated here and
+	// in AdminCreationParams, might as well just use them from there
+	RPCSecret                    string
+	AdminToken                   string
 	GlobalBucketS3APICredentials garage.S3APICredentials
 }
 

go/bootstrap/garage.go

@@ -27,6 +27,17 @@ func (b Bootstrap) GaragePeers() []garage.RemotePeer {
 	return peers
 }
 
+// GarageRPCPeerAddrs returns the full RPC peer address for each known garage
+// instance in the network.
+func (b Bootstrap) GarageRPCPeerAddrs() []string {
+	var addrs []string
+	for _, peer := range b.GaragePeers() {
+		addr := peer.RPCPeerAddr()
+		addrs = append(addrs, addr)
+	}
+	return addrs
+}
+
 // ChooseGaragePeer returns a Peer for a garage instance from the network. It
 // will prefer a garage instance on this particular host, if there is one, but
 // will otherwise return a random endpoint.

go/bootstrap/hosts.go

@@ -78,5 +78,10 @@ type Host struct {
 // This assumes that the Host and its data has already been verified against the
 // CA signing key.
 func (h Host) IP() net.IP {
-	return h.PublicCredentials.Cert.Details.Ips[0].IP
+	ip, err := nebula.IPFromHostCertPEM(h.PublicCredentials.CertPEM)
+	if err != nil {
+		panic(fmt.Errorf("could not parse IP out of cert for host %q: %w", h.Name, err))
+	}
+
+	return ip
 }

go/cmd/entrypoint/admin.go

@@ -149,8 +149,9 @@ var subCmdAdminCreateNetwork = subCmd{
 		}
 
 		garageBootstrap := bootstrap.Garage{
-			RPCSecret:  randStr(32),
-			AdminToken: randStr(32),
+			RPCSecret:                    randStr(32),
+			AdminToken:                   randStr(32),
+			GlobalBucketS3APICredentials: garage.NewS3APICredentials(),
 		}
 
 		hostBootstrap, err := bootstrap.New(
@@ -215,17 +216,7 @@ var subCmdAdminCreateNetwork = subCmd{
 		}
 
 		logger.Info(ctx, "initializing garage shared global bucket")
-		garageGlobalBucketCreds, err := garageInitializeGlobalBucket(
-			ctx, logger, hostBootstrap, daemonConfig,
-		)
-
-		hostBootstrap.Garage.GlobalBucketS3APICredentials = garageGlobalBucketCreds
-
-		// rewrite the bootstrap now that the global bucket creds have been
-		// added to it.
-		if err := writeBootstrapToDataDir(hostBootstrap); err != nil {
-			return fmt.Errorf("writing bootstrap file: %w", err)
-		}
+		err = garageInitializeGlobalBucket(ctx, logger, hostBootstrap, daemonConfig)
 
 		if cErr := (garage.AdminClientError{}); errors.As(err, &cErr) && cErr.StatusCode == 409 {
 			return fmt.Errorf("shared global bucket has already been created, are the storage allocations from a previously initialized isle being used?")
@@ -381,23 +372,13 @@ var subCmdAdminCreateNebulaCert = subCmd{
 			return fmt.Errorf("reading public key from %q: %w", *pubKeyPath, err)
 		}
 
-		var hostPub nebula.EncryptingPublicKey
-		if err := hostPub.UnmarshalNebulaPEM(hostPubPEM); err != nil {
-			return fmt.Errorf("unmarshaling public key as PEM: %w", err)
-		}
-
-		nebulaHostCert, err := nebula.NewHostCert(
-			adm.Nebula.CACredentials, hostPub, *hostName, ip,
+		nebulaHostCertPEM, err := nebula.NewHostCertPEM(
+			adm.Nebula.CACredentials, string(hostPubPEM), *hostName, ip,
 		)
 		if err != nil {
 			return fmt.Errorf("creating cert: %w", err)
 		}
 
-		nebulaHostCertPEM, err := nebulaHostCert.MarshalToPEM()
-		if err != nil {
-			return fmt.Errorf("marshaling cert to PEM: %w", err)
-		}
-
 		if _, err := os.Stdout.Write([]byte(nebulaHostCertPEM)); err != nil {
 			return fmt.Errorf("writing to stdout: %w", err)
 		}

go/cmd/entrypoint/daemon_util.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"isle/bootstrap"
 	"isle/daemon"
-	"isle/garage/garagesrv"
+	"isle/garage"
 	"time"
 )
 
@@ -29,7 +29,7 @@ func coalesceDaemonConfigAndBootstrap(
 
 		for i, alloc := range allocs {
 
-			id, rpcPort, err := garagesrv.InitAlloc(alloc.MetaPath, alloc.RPCPort)
+			id, rpcPort, err := garage.InitAlloc(alloc.MetaPath, alloc.RPCPort)
 			if err != nil {
 				return bootstrap.Bootstrap{}, daemon.Config{}, fmt.Errorf("initializing alloc at %q: %w", alloc.MetaPath, err)
 			}

go/cmd/entrypoint/garage_util.go

@@ -2,11 +2,10 @@ package main
 
 import (
 	"context"
-	"fmt"
 	"isle/bootstrap"
 	"isle/daemon"
 	"isle/garage"
-	"isle/garage/garagesrv"
+	"fmt"
 	"net"
 	"path/filepath"
 	"strconv"
@@ -31,12 +30,12 @@ func newGarageAdminClient(
 	thisHost := hostBootstrap.ThisHost()
 
 	return garage.NewAdminClient(
-		garageAdminClientLogger(logger),
 		net.JoinHostPort(
 			thisHost.IP().String(),
 			strconv.Itoa(daemonConfig.Storage.Allocations[0].AdminPort),
 		),
 		hostBootstrap.Garage.AdminToken,
+		garageAdminClientLogger(logger),
 	)
 }
 
@@ -69,9 +68,9 @@ func waitForGarageAndNebula(
 		)
 
 		adminClient := garage.NewAdminClient(
-			adminClientLogger,
 			adminAddr,
 			hostBootstrap.Garage.AdminToken,
+			adminClientLogger,
 		)
 
 		ctx := mctx.Annotate(ctx, "garageAdminAddr", adminAddr)
@@ -129,15 +128,18 @@ func garageWriteChildConfig(
 		envRuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
 	)
 
-	err := garagesrv.WriteGarageTomlFile(garageTomlPath, garagesrv.GarageTomlData{
+	err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
 		MetaPath: alloc.MetaPath,
 		DataPath: alloc.DataPath,
 
 		RPCSecret:  hostBootstrap.Garage.RPCSecret,
 		AdminToken: hostBootstrap.Garage.AdminToken,
 
-		LocalPeer:      peer,
-		BootstrapPeers: hostBootstrap.GaragePeers(),
+		RPCAddr:   peer.RPCAddr(),
+		S3APIAddr: peer.S3APIAddr(),
+		AdminAddr: peer.AdminAddr(),
+
+		BootstrapPeers: hostBootstrap.GarageRPCPeerAddrs(),
 	})
 
 	if err != nil {
@@ -182,36 +184,62 @@ func garageInitializeGlobalBucket(
 	logger *mlog.Logger,
 	hostBootstrap bootstrap.Bootstrap,
 	daemonConfig daemon.Config,
-) (
-	garage.S3APICredentials, error,
-) {
-	adminClient := newGarageAdminClient(logger, hostBootstrap, daemonConfig)
+) error {
 
-	creds, err := adminClient.CreateS3APICredentials(
-		ctx, garage.GlobalBucketS3APICredentialsName,
+	var (
+		adminClient       = newGarageAdminClient(logger, hostBootstrap, daemonConfig)
+		globalBucketCreds = hostBootstrap.Garage.GlobalBucketS3APICredentials
 	)
+
+	// first attempt to import the key
+	err := adminClient.Do(ctx, nil, "POST", "/v0/key/import", map[string]string{
+		"accessKeyId":     globalBucketCreds.ID,
+		"secretAccessKey": globalBucketCreds.Secret,
+		"name":            "shared-global-bucket-key",
+	})
+
 	if err != nil {
-		return creds, fmt.Errorf("creating global bucket credentials: %w", err)
+		return fmt.Errorf("importing global bucket key into garage: %w", err)
 	}
 
-	bucketID, err := adminClient.CreateBucket(ctx, garage.GlobalBucket)
+	// create global bucket
+	err = adminClient.Do(ctx, nil, "POST", "/v0/bucket", map[string]string{
+		"globalAlias": garage.GlobalBucket,
+	})
+
 	if err != nil {
-		return creds, fmt.Errorf("creating global bucket: %w", err)
+		return fmt.Errorf("creating global bucket: %w", err)
 	}
 
-	if err := adminClient.GrantBucketPermissions(
-		ctx,
-		bucketID,
-		creds.ID,
-		garage.BucketPermissionRead,
-		garage.BucketPermissionWrite,
-	); err != nil {
-		return creds, fmt.Errorf(
-			"granting permissions to shared global bucket key: %w", err,
-		)
+	// retrieve newly created bucket's id
+	var getBucketRes struct {
+		ID string `json:"id"`
 	}
 
-	return creds, nil
+	err = adminClient.Do(
+		ctx, &getBucketRes,
+		"GET", "/v0/bucket?globalAlias="+garage.GlobalBucket, nil,
+	)
+
+	if err != nil {
+		return fmt.Errorf("fetching global bucket id: %w", err)
+	}
+
+	// allow shared global bucket key to perform all operations
+	err = adminClient.Do(ctx, nil, "POST", "/v0/bucket/allow", map[string]interface{}{
+		"bucketId":    getBucketRes.ID,
+		"accessKeyId": globalBucketCreds.ID,
+		"permissions": map[string]bool{
+			"read":  true,
+			"write": true,
+		},
+	})
+
+	if err != nil {
+		return fmt.Errorf("granting permissions to shared global bucket key: %w", err)
+	}
+
+	return nil
 }
 
 func garageApplyLayout(
@@ -226,25 +254,62 @@ func garageApplyLayout(
 		thisHost    = hostBootstrap.ThisHost()
 		hostName    = thisHost.Name
 		allocs      = daemonConfig.Storage.Allocations
-		peers       = make([]garage.PeerLayout, len(allocs))
 	)
 
-	for i, alloc := range allocs {
+	type peerLayout struct {
+		Capacity int      `json:"capacity"`
+		Zone     string   `json:"zone"`
+		Tags     []string `json:"tags"`
+	}
 
-		id := bootstrapGarageHostForAlloc(thisHost, alloc).ID
+	{
+		clusterLayout := map[string]peerLayout{}
 
-		zone := hostName
-		if alloc.Zone != "" {
-			zone = alloc.Zone
+		for _, alloc := range allocs {
+
+			id := bootstrapGarageHostForAlloc(thisHost, alloc).ID
+
+			zone := hostName
+			if alloc.Zone != "" {
+				zone = alloc.Zone
+			}
+
+			clusterLayout[id] = peerLayout{
+				Capacity: alloc.Capacity,
+				Zone:     zone,
+				Tags:     []string{},
+			}
 		}
 
-		peers[i] = garage.PeerLayout{
-			ID:       id,
-			Capacity: alloc.Capacity * 1_000_000_000,
-			Zone:     zone,
-			Tags:     []string{},
+		err := adminClient.Do(ctx, nil, "POST", "/v0/layout", clusterLayout)
+		if err != nil {
+			return fmt.Errorf("staging layout changes: %w", err)
 		}
 	}
 
-	return adminClient.ApplyLayout(ctx, peers)
+	var clusterLayout struct {
+		Version           int                   `json:"version"`
+		StagedRoleChanges map[string]peerLayout `json:"stagedRoleChanges"`
+	}
+
+	if err := adminClient.Do(ctx, &clusterLayout, "GET", "/v0/layout", nil); err != nil {
+		return fmt.Errorf("retrieving staged layout change: %w", err)
+	}
+
+	if len(clusterLayout.StagedRoleChanges) == 0 {
+		return nil
+	}
+
+	applyClusterLayout := struct {
+		Version int `json:"version"`
+	}{
+		Version: clusterLayout.Version + 1,
+	}
+
+	err := adminClient.Do(ctx, nil, "POST", "/v0/layout/apply", applyClusterLayout)
+	if err != nil {
+		return fmt.Errorf("applying new layout (new version:%d): %w", applyClusterLayout.Version, err)
+	}
+
+	return nil
 }

go/cmd/entrypoint/nebula.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"isle/jsonutil"
 	"os"
+
+	"github.com/slackhq/nebula/cert"
 )
 
 var subCmdNebulaShow = subCmd{
@@ -21,10 +23,10 @@ var subCmdNebulaShow = subCmd{
 			return fmt.Errorf("loading host bootstrap: %w", err)
 		}
 
-		caCert := hostBootstrap.CAPublicCredentials.Cert
-		caCertPEM, err := caCert.MarshalToPEM()
+		caPublicCreds := hostBootstrap.CAPublicCredentials
+		caCert, _, err := cert.UnmarshalNebulaCertificateFromPEM([]byte(caPublicCreds.CertPEM))
 		if err != nil {
-			return fmt.Errorf("marshaling CA cert to PEM: %w", err)
+			return fmt.Errorf("unmarshaling ca.crt: %w", err)
 		}
 
 		if len(caCert.Details.Subnets) != 1 {
@@ -46,7 +48,7 @@ var subCmdNebulaShow = subCmd{
 			SubnetCIDR  string
 			Lighthouses []outLighthouse
 		}{
-			CACert:     string(caCertPEM),
+			CACert:     caPublicCreds.CertPEM,
 			SubnetCIDR: subnet.String(),
 		}
 

go/cmd/entrypoint/nebula_util.go

@@ -10,7 +10,6 @@ import (
 	"path/filepath"
 
 	"code.betamike.com/micropelago/pmux/pmuxlib"
-	"github.com/slackhq/nebula/cert"
 )
 
 // waitForNebula waits for the nebula interface to have been started up. It does
@@ -57,29 +56,11 @@ func nebulaPmuxProcConfig(
 		staticHostMap[ip] = []string{host.Nebula.PublicAddr}
 	}
 
-	caCertPEM, err := hostBootstrap.CAPublicCredentials.Cert.MarshalToPEM()
-	if err != nil {
-		return pmuxlib.ProcessConfig{}, fmt.Errorf(
-			"marshaling CA cert to PEM: :%w", err,
-		)
-	}
-
-	hostCertPEM, err := hostBootstrap.PublicCredentials.Cert.MarshalToPEM()
-	if err != nil {
-		return pmuxlib.ProcessConfig{}, fmt.Errorf(
-			"marshaling host cert to PEM: :%w", err,
-		)
-	}
-
-	hostKeyPEM := cert.MarshalX25519PrivateKey(
-		hostBootstrap.PrivateCredentials.EncryptingPrivateKey.Bytes(),
-	)
-
 	config := map[string]interface{}{
 		"pki": map[string]string{
-			"ca":   string(caCertPEM),
-			"cert": string(hostCertPEM),
-			"key":  string(hostKeyPEM),
+			"ca":   hostBootstrap.CAPublicCredentials.CertPEM,
+			"cert": hostBootstrap.PublicCredentials.CertPEM,
+			"key":  hostBootstrap.PrivateCredentials.PrivateKeyPEM,
 		},
 		"static_host_map": staticHostMap,
 		"punchy": map[string]bool{

go/garage/admin_client.go

@@ -14,22 +14,7 @@ import (
 	"github.com/mediocregopher/mediocre-go-lib/v2/mlog"
 )
 
-// BucketID is a unique identifier for a bucket in garage. It is different than
-// the bucket's alias, since a bucket may have multiple aliases.
-type BucketID string
-
-// BucketPermission describes a permission an S3APICredentials may have when
-// interacting with a bucket.
-type BucketPermission string
-
-// Enumeration of BucketPermissions.
-const (
-	BucketPermissionRead  BucketPermission = "read"
-	BucketPermissionWrite BucketPermission = "write"
-	BucketPermissionOwner BucketPermission = "owner"
-)
-
-// AdminClientError gets returned from AdminClient Do methods for non-200
+// AdminClientError gets returned from AdminClient's Do method for non-200
 // errors.
 type AdminClientError struct {
 	StatusCode int
@@ -43,32 +28,32 @@ func (e AdminClientError) Error() string {
 // AdminClient is a helper type for performing actions against the garage admin
 // interface.
 type AdminClient struct {
-	logger     *mlog.Logger
 	c          *http.Client
 	addr       string
 	adminToken string
+	logger     *mlog.Logger
 }
 
 // NewAdminClient initializes and returns an AdminClient which will use the
 // given address and adminToken for all requests made.
 //
 // If Logger is nil then logs will be suppressed.
-func NewAdminClient(logger *mlog.Logger, addr, adminToken string) *AdminClient {
+func NewAdminClient(addr, adminToken string, logger *mlog.Logger) *AdminClient {
 	return &AdminClient{
-		logger: logger,
 		c: &http.Client{
 			Transport: http.DefaultTransport.(*http.Transport).Clone(),
 		},
 		addr:       addr,
 		adminToken: adminToken,
+		logger:     logger,
 	}
 }
 
-// do performs an HTTP request with the given method (GET, POST) and path, and
+// Do performs an HTTP request with the given method (GET, POST) and path, and
 // using the json marshaling of the given body as the request body (unless body
 // is nil). It will JSON unmarshal the response into rcv, unless rcv is nil.
-func (c *AdminClient) do(
-	ctx context.Context, rcv any, method, path string, body any,
+func (c *AdminClient) Do(
+	ctx context.Context, rcv interface{}, method, path string, body interface{},
 ) error {
 
 	var bodyR io.Reader
@@ -127,6 +112,7 @@ func (c *AdminClient) do(
 	}
 
 	if rcv == nil {
+
 		if _, err := io.Copy(io.Discard, res.Body); err != nil {
 			return fmt.Errorf("discarding response body: %w", err)
 		}
@@ -152,14 +138,13 @@ func (c *AdminClient) Wait(ctx context.Context) error {
 			time.Sleep(250 * time.Millisecond)
 		}
 
-		// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Nodes/operation/GetNodes
 		var clusterStatus struct {
-			Nodes []struct {
-				IsUp bool `json:"isUp"`
-			} `json:"nodes"`
+			KnownNodes map[string]struct {
+				IsUp bool `json:"is_up"`
+			} `json:"knownNodes"`
 		}
 
-		err := c.do(ctx, &clusterStatus, "GET", "/v1/status", nil)
+		err := c.Do(ctx, &clusterStatus, "GET", "/v0/status", nil)
 
 		if ctxErr := ctx.Err(); ctxErr != nil {
 			return ctxErr
@@ -171,14 +156,14 @@ func (c *AdminClient) Wait(ctx context.Context) error {
 
 		var numUp int
 
-		for _, node := range clusterStatus.Nodes {
-			if node.IsUp {
+		for _, knownNode := range clusterStatus.KnownNodes {
+			if knownNode.IsUp {
 				numUp++
 			}
 		}
 
 		ctx := mctx.Annotate(ctx,
-			"numNodes", len(clusterStatus.Nodes),
+			"numKnownNodes", len(clusterStatus.KnownNodes),
 			"numUp", numUp,
 		)
 
@@ -190,134 +175,3 @@ func (c *AdminClient) Wait(ctx context.Context) error {
 		c.logger.Debug(ctx, "instance not online yet, will continue waiting")
 	}
 }
-
-// CreateS3APICredentials creates an S3APICredentials with the given name. The
-// credentials must be associated with a bucket via a call to
-// GrantBucketPermissions in order to be useful.
-func (c *AdminClient) CreateS3APICredentials(
-	ctx context.Context, name string,
-) (
-	S3APICredentials, error,
-) {
-	// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Key/operation/AddKey
-
-	var res struct {
-		ID     string `json:"accessKeyId"`
-		Secret string `json:"secretAccessKey"`
-	}
-
-	// first attempt to import the key
-	err := c.do(
-		ctx, &res, "POST", "/v1/key", map[string]string{"name": name},
-	)
-
-	return S3APICredentials{
-		ID:     res.ID,
-		Secret: res.Secret,
-	}, err
-}
-
-// CreateBucket creates a bucket with the given global alias, returning its ID.
-func (c *AdminClient) CreateBucket(
-	ctx context.Context, globalAlias string,
-) (
-	BucketID, error,
-) {
-	// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Bucket/operation/PutBucketGlobalAlias
-
-	var res struct {
-		ID string `json:"id"`
-	}
-
-	err := c.do(
-		ctx, &res, "POST", "/v1/bucket", map[string]string{
-			"globalAlias": globalAlias,
-		},
-	)
-
-	return BucketID(res.ID), err
-}
-
-// GrantBucketPermissions grants the S3APICredentials with the given ID
-// permission(s) to interact with the bucket of the given ID.
-func (c *AdminClient) GrantBucketPermissions(
-	ctx context.Context,
-	bucketID BucketID,
-	credentialsID string,
-	perms ...BucketPermission,
-) error {
-	permsMap := map[BucketPermission]bool{}
-	for _, perm := range perms {
-		permsMap[perm] = true
-	}
-
-	// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Bucket/operation/AllowBucketKey
-	return c.do(ctx, nil, "POST", "/v1/bucket/allow", map[string]any{
-		"bucketId":    bucketID,
-		"accessKeyId": credentialsID,
-		"permissions": permsMap,
-	})
-}
-
-// PeerLayout describes the properties of a garage peer in the context of the
-// layout of the cluster.
-type PeerLayout struct {
-	ID       string
-	Capacity int // Gb (SI units)
-	Zone     string
-	Tags     []string
-}
-
-// ApplyLayout modifies the layout of the garage cluster. Only layout of the
-// given peers will be modified/created, other peers are not affected.
-func (c *AdminClient) ApplyLayout(
-	ctx context.Context, peers []PeerLayout,
-) error {
-	type peerLayout struct {
-		ID       string   `json:"id"`
-		Capacity int      `json:"capacity"`
-		Zone     string   `json:"zone"`
-		Tags     []string `json:"tags"`
-	}
-
-	{
-		// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Layout/operation/ApplyLayout
-		clusterLayout := make([]peerLayout, len(peers))
-		for i := range peers {
-			clusterLayout[i] = peerLayout(peers[i])
-		}
-
-		err := c.do(ctx, nil, "POST", "/v1/layout", clusterLayout)
-		if err != nil {
-			return fmt.Errorf("staging layout changes: %w", err)
-		}
-	}
-
-	// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Layout/operation/GetLayout
-	var clusterLayout struct {
-		Version           int          `json:"version"`
-		StagedRoleChanges []peerLayout `json:"stagedRoleChanges"`
-	}
-
-	if err := c.do(ctx, &clusterLayout, "GET", "/v1/layout", nil); err != nil {
-		return fmt.Errorf("retrieving staged layout change: %w", err)
-	}
-
-	if len(clusterLayout.StagedRoleChanges) == 0 {
-		return nil
-	}
-
-	// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Layout/operation/ApplyLayout
-	applyClusterLayout := struct {
-		Version int `json:"version"`
-	}{
-		Version: clusterLayout.Version + 1,
-	}
-
-	err := c.do(ctx, nil, "POST", "/v1/layout/apply", applyClusterLayout)
-	if err != nil {
-		return fmt.Errorf("applying new layout (new version:%d): %w", applyClusterLayout.Version, err)
-	}
-
-	return nil
-}

go/garage/client.go

@@ -1,6 +1,8 @@
 package garage
 
 import (
+	"crypto/rand"
+	"encoding/hex"
 	"errors"
 	"fmt"
 
@@ -8,6 +10,14 @@ import (
 	"github.com/minio/minio-go/v7/pkg/credentials"
 )
 
+func randStr(l int) string {
+	b := make([]byte, l)
+	if _, err := rand.Read(b); err != nil {
+		panic(err)
+	}
+	return hex.EncodeToString(b)
+}
+
 // IsKeyNotFound returns true if the given error is the result of a key not
 // being found in a bucket.
 func IsKeyNotFound(err error) bool {
@@ -25,6 +35,14 @@ type S3APICredentials struct {
 	Secret string
 }
 
+// NewS3APICredentials returns a new usable instance of S3APICredentials.
+func NewS3APICredentials() S3APICredentials {
+	return S3APICredentials{
+		ID:     randStr(8),
+		Secret: randStr(32),
+	}
+}
+
 // NewS3APIClient returns a minio client configured to use the given garage S3 API
 // endpoint.
 func NewS3APIClient(addr string, creds S3APICredentials) S3APIClient {

go/garage/garage.go

@@ -1,7 +1,17 @@
-// Package garage contains types and helpers related to interacting with garage
-// processes via garage's APIs.
+// Package garage contains helper functions and types which are useful for
+// setting up garage configs, processes, and deployments.
 package garage
 
+import (
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+)
+
 const (
 
 	// Region is the region which garage is configured with.
@@ -11,9 +21,133 @@ const (
 	// accessible to all hosts in the network.
 	GlobalBucket = "global-shared"
 
-	GlobalBucketS3APICredentialsName = "global-shared-key"
-
 	// ReplicationFactor indicates the replication factor set on the garage
 	// cluster. We currently only support a factor of 3.
 	ReplicationFactor = 3
 )
+
+func nodeKeyPath(metaDirPath string) string {
+	return filepath.Join(metaDirPath, "node_key")
+}
+
+func nodeKeyPubPath(metaDirPath string) string {
+	return filepath.Join(metaDirPath, "node_key.pub")
+}
+
+func nodeRPCPortPath(metaDirPath string) string {
+	return filepath.Join(metaDirPath, "isle", "rpc_port")
+}
+
+// loadAllocID returns the peer ID (ie the public key) of the node at the given
+// meta directory.
+func loadAllocID(metaDirPath string) (string, error) {
+	nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
+
+	pubKey, err := os.ReadFile(nodeKeyPubPath)
+	if err != nil {
+		return "", fmt.Errorf("reading %q: %w", nodeKeyPubPath, err)
+	}
+
+	return hex.EncodeToString(pubKey), nil
+}
+
+// InitAlloc initializes the meta directory and keys for a particular
+// allocation, if it hasn't been done so already. It returns the peer ID (ie the
+// public key) and the rpc port in any case.
+func InitAlloc(metaDirPath string, initRPCPort int) (string, int, error) {
+
+	initDirFor := func(path string) error {
+		dir := filepath.Dir(path)
+		return os.MkdirAll(dir, 0750)
+	}
+
+	var err error
+
+	exists := func(path string) bool {
+
+		if err != nil {
+			return false
+
+		} else if _, err = os.Stat(path); errors.Is(err, fs.ErrNotExist) {
+			err = nil
+			return false
+
+		} else if err != nil {
+			err = fmt.Errorf("checking if %q exists: %w", path, err)
+			return false
+		}
+
+		return true
+	}
+
+	nodeKeyPath := nodeKeyPath(metaDirPath)
+	nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
+	nodeRPCPortPath := nodeRPCPortPath(metaDirPath)
+
+	nodeKeyPathExists := exists(nodeKeyPath)
+	nodeKeyPubPathExists := exists(nodeKeyPubPath)
+	nodeRPCPortPathExists := exists(nodeRPCPortPath)
+
+	if err != nil {
+		return "", 0, err
+
+	} else if nodeKeyPubPathExists != nodeKeyPathExists {
+		return "", 0, fmt.Errorf("%q or %q exist without the other existing", nodeKeyPath, nodeKeyPubPath)
+
+	}
+
+	var (
+		pubKeyStr string
+		rpcPort   int
+	)
+
+	if nodeKeyPathExists {
+
+		if pubKeyStr, err = loadAllocID(metaDirPath); err != nil {
+			return "", 0, fmt.Errorf("reading node public key file: %w", err)
+		}
+
+	} else {
+
+		if err := initDirFor(nodeKeyPath); err != nil {
+			return "", 0, fmt.Errorf("creating directory for %q: %w", nodeKeyPath, err)
+		}
+
+		pubKey, privKey := GeneratePeerKey()
+
+		if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
+			return "", 0, fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
+
+		} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
+			return "", 0, fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
+		}
+
+		pubKeyStr = hex.EncodeToString(pubKey)
+	}
+
+	if nodeRPCPortPathExists {
+
+		if rpcPortStr, err := os.ReadFile(nodeRPCPortPath); err != nil {
+			return "", 0, fmt.Errorf("reading rpc port from %q: %w", nodeRPCPortPath, err)
+
+		} else if rpcPort, err = strconv.Atoi(string(rpcPortStr)); err != nil {
+			return "", 0, fmt.Errorf("parsing rpc port %q from %q: %w", rpcPortStr, nodeRPCPortPath, err)
+		}
+
+	} else {
+
+		if err := initDirFor(nodeRPCPortPath); err != nil {
+			return "", 0, fmt.Errorf("creating directory for %q: %w", nodeRPCPortPath, err)
+		}
+
+		rpcPortStr := strconv.Itoa(initRPCPort)
+
+		if err := os.WriteFile(nodeRPCPortPath, []byte(rpcPortStr), 0440); err != nil {
+			return "", 0, fmt.Errorf("writing rpc port %q to %q: %w", rpcPortStr, nodeRPCPortPath, err)
+		}
+
+		rpcPort = initRPCPort
+	}
+
+	return pubKeyStr, rpcPort, nil
+}

go/garage/garagesrv/garagesrv.go

@@ -1,152 +0,0 @@
-// Package garage contains helper functions and types which are useful for
-// setting up garage configs, processes, and deployments.
-package garagesrv
-
-import (
-	"crypto/ed25519"
-	"crypto/rand"
-	"encoding/hex"
-	"errors"
-	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"strconv"
-)
-
-func nodeKeyPath(metaDirPath string) string {
-	return filepath.Join(metaDirPath, "node_key")
-}
-
-func nodeKeyPubPath(metaDirPath string) string {
-	return filepath.Join(metaDirPath, "node_key.pub")
-}
-
-func nodeRPCPortPath(metaDirPath string) string {
-	return filepath.Join(metaDirPath, "isle", "rpc_port")
-}
-
-// loadAllocID returns the peer ID (ie the public key) of the node at the given
-// meta directory.
-func loadAllocID(metaDirPath string) (string, error) {
-	nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
-
-	pubKey, err := os.ReadFile(nodeKeyPubPath)
-	if err != nil {
-		return "", fmt.Errorf("reading %q: %w", nodeKeyPubPath, err)
-	}
-
-	return hex.EncodeToString(pubKey), nil
-}
-
-// generatePeerKey generates and returns a public/private key pair for a garage
-// instance.
-func generatePeerKey() (pubKey, privKey []byte) {
-	pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
-	if err != nil {
-		panic(err)
-	}
-
-	return pubKey, privKey
-}
-
-// InitAlloc initializes the meta directory and keys for a particular
-// allocation, if it hasn't been done so already. It returns the peer ID (ie the
-// public key) and the rpc port in any case.
-func InitAlloc(metaDirPath string, initRPCPort int) (string, int, error) {
-
-	initDirFor := func(path string) error {
-		dir := filepath.Dir(path)
-		return os.MkdirAll(dir, 0750)
-	}
-
-	var err error
-
-	exists := func(path string) bool {
-
-		if err != nil {
-			return false
-
-		} else if _, err = os.Stat(path); errors.Is(err, fs.ErrNotExist) {
-			err = nil
-			return false
-
-		} else if err != nil {
-			err = fmt.Errorf("checking if %q exists: %w", path, err)
-			return false
-		}
-
-		return true
-	}
-
-	nodeKeyPath := nodeKeyPath(metaDirPath)
-	nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
-	nodeRPCPortPath := nodeRPCPortPath(metaDirPath)
-
-	nodeKeyPathExists := exists(nodeKeyPath)
-	nodeKeyPubPathExists := exists(nodeKeyPubPath)
-	nodeRPCPortPathExists := exists(nodeRPCPortPath)
-
-	if err != nil {
-		return "", 0, err
-
-	} else if nodeKeyPubPathExists != nodeKeyPathExists {
-		return "", 0, fmt.Errorf("%q or %q exist without the other existing", nodeKeyPath, nodeKeyPubPath)
-
-	}
-
-	var (
-		pubKeyStr string
-		rpcPort   int
-	)
-
-	if nodeKeyPathExists {
-
-		if pubKeyStr, err = loadAllocID(metaDirPath); err != nil {
-			return "", 0, fmt.Errorf("reading node public key file: %w", err)
-		}
-
-	} else {
-
-		if err := initDirFor(nodeKeyPath); err != nil {
-			return "", 0, fmt.Errorf("creating directory for %q: %w", nodeKeyPath, err)
-		}
-
-		pubKey, privKey := generatePeerKey()
-
-		if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
-			return "", 0, fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
-
-		} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
-			return "", 0, fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
-		}
-
-		pubKeyStr = hex.EncodeToString(pubKey)
-	}
-
-	if nodeRPCPortPathExists {
-
-		if rpcPortStr, err := os.ReadFile(nodeRPCPortPath); err != nil {
-			return "", 0, fmt.Errorf("reading rpc port from %q: %w", nodeRPCPortPath, err)
-
-		} else if rpcPort, err = strconv.Atoi(string(rpcPortStr)); err != nil {
-			return "", 0, fmt.Errorf("parsing rpc port %q from %q: %w", rpcPortStr, nodeRPCPortPath, err)
-		}
-
-	} else {
-
-		if err := initDirFor(nodeRPCPortPath); err != nil {
-			return "", 0, fmt.Errorf("creating directory for %q: %w", nodeRPCPortPath, err)
-		}
-
-		rpcPortStr := strconv.Itoa(initRPCPort)
-
-		if err := os.WriteFile(nodeRPCPortPath, []byte(rpcPortStr), 0440); err != nil {
-			return "", 0, fmt.Errorf("writing rpc port %q to %q: %w", rpcPortStr, nodeRPCPortPath, err)
-		}
-
-		rpcPort = initRPCPort
-	}
-
-	return pubKeyStr, rpcPort, nil
-}

go/garage/peer.go

@@ -1,6 +1,8 @@
 package garage
 
 import (
+	"crypto/ed25519"
+	"crypto/rand"
 	"fmt"
 	"net"
 	"strconv"
@@ -22,6 +24,17 @@ type LocalPeer struct {
 	AdminPort int
 }
 
+// GeneratePeerKey generates and returns a public/private key pair for a garage
+// instance.
+func GeneratePeerKey() (pubKey, privKey []byte) {
+	pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		panic(err)
+	}
+
+	return pubKey, privKey
+}
+
 // RPCAddr returns the address of the peer's RPC port.
 func (p RemotePeer) RPCAddr() string {
 	return net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort))

go/garage/tpl.go

@@ -1,13 +1,10 @@
-package garagesrv
+package garage
 
 import (
 	"fmt"
 	"io"
 	"os"
-	"strconv"
 	"text/template"
-
-	"isle/garage"
 )
 
 // GarageTomlData describes all fields needed for rendering a garage.toml
@@ -19,8 +16,11 @@ type GarageTomlData struct {
 	RPCSecret  string
 	AdminToken string
 
-	garage.LocalPeer
-	BootstrapPeers []garage.RemotePeer
+	RPCAddr   string
+	S3APIAddr string
+	AdminAddr string
+
+	BootstrapPeers []string
 }
 
 var garageTomlTpl = template.Must(template.New("").Parse(`
@@ -28,14 +28,14 @@ var garageTomlTpl = template.Must(template.New("").Parse(`
 metadata_dir = "{{ .MetaPath }}"
 data_dir = "{{ .DataPath }}"
 
-replication_mode = "` + strconv.Itoa(garage.ReplicationFactor) + `"
+replication_mode = "3"
 
 rpc_secret = "{{ .RPCSecret }}"
 rpc_bind_addr = "{{ .RPCAddr }}"
 rpc_public_addr = "{{ .RPCAddr }}"
 
 bootstrap_peers = [{{- range .BootstrapPeers }}
-"{{ .RPCPeerAddr }}",
+"{{ . }}",
 {{ end -}}]
 
 [s3_api]

go/go.mod

@@ -23,7 +23,6 @@ require (
 	github.com/gopherjs/gopherjs v1.17.2 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/jtolds/gls v4.20.0+incompatible // indirect
-	github.com/jxskiss/base62 v1.1.0 // indirect
 	github.com/klauspost/compress v1.13.5 // indirect
 	github.com/klauspost/cpuid v1.3.1 // indirect
 	github.com/minio/md5-simd v1.1.0 // indirect

go/go.sum

@@ -24,8 +24,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw=
-github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc=
 github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4=
 github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=

go/nebula/encrypting_key.go

@@ -1,117 +0,0 @@
-package nebula
-
-import (
-	"crypto/ecdh"
-	"crypto/rand"
-	"fmt"
-
-	"github.com/slackhq/nebula/cert"
-)
-
-var (
-	encPrivKeyPrefix = []byte("x0")
-	encPubKeyPrefix  = []byte("X0")
-
-	x25519 = ecdh.X25519()
-)
-
-// EncryptingPublicKey wraps an X25519-based ECDH public key to provide
-// convenient text (un)marshaling methods.
-type EncryptingPublicKey struct{ inner *ecdh.PublicKey }
-
-// MarshalText implements the encoding.TextMarshaler interface.
-func (pk EncryptingPublicKey) MarshalText() ([]byte, error) {
-	return encodeWithPrefix(encPubKeyPrefix, pk.inner.Bytes()), nil
-}
-
-// Bytes returns the raw bytes of the EncryptingPublicKey.
-func (k EncryptingPublicKey) Bytes() []byte {
-	return k.inner.Bytes()
-}
-
-// UnmarshalText implements the encoding.TextUnmarshaler interface.
-func (pk *EncryptingPublicKey) UnmarshalText(b []byte) error {
-	b, err := decodeWithPrefix(encPubKeyPrefix, b)
-	if err != nil {
-		return fmt.Errorf("unmarshaling: %w", err)
-	}
-
-	if pk.inner, err = x25519.NewPublicKey(b); err != nil {
-		return fmt.Errorf("converting bytes to public key: %w", err)
-	}
-
-	return nil
-}
-
-// UnmarshalNebulaPEM unmarshals the EncryptingPublicKey as a nebula host public
-// key PEM.
-func (pk *EncryptingPublicKey) UnmarshalNebulaPEM(b []byte) error {
-	b, _, err := cert.UnmarshalEd25519PublicKey(b)
-	if err != nil {
-		return fmt.Errorf("unmarshaling: %w", err)
-	}
-
-	if pk.inner, err = x25519.NewPublicKey(b); err != nil {
-		return fmt.Errorf("converting bytes to public key: %w", err)
-	}
-
-	return nil
-}
-
-func (pk EncryptingPublicKey) String() string {
-	b, err := pk.MarshalText()
-	if err != nil {
-		panic(err)
-	}
-	return string(b)
-}
-
-// EncryptingPrivateKey wraps an X25519-based ECDH private key to provide
-// convenient text (un)marshaling methods.
-type EncryptingPrivateKey struct{ inner *ecdh.PrivateKey }
-
-// NewEncryptingPrivateKey generates and returns a fresh EncryptingPrivateKey.
-func NewEncryptingPrivateKey() EncryptingPrivateKey {
-	k, err := x25519.GenerateKey(rand.Reader)
-	if err != nil {
-		panic(err)
-	}
-	return EncryptingPrivateKey{k}
-}
-
-// PublicKey returns the public key which corresponds with this private key.
-func (k EncryptingPrivateKey) PublicKey() EncryptingPublicKey {
-	return EncryptingPublicKey{k.inner.PublicKey()}
-}
-
-// MarshalText implements the encoding.TextMarshaler interface.
-func (k EncryptingPrivateKey) MarshalText() ([]byte, error) {
-	return encodeWithPrefix(encPrivKeyPrefix, k.inner.Bytes()), nil
-}
-
-// Bytes returns the raw bytes of the EncryptingPrivateKey.
-func (k EncryptingPrivateKey) Bytes() []byte {
-	return k.inner.Bytes()
-}
-
-// UnmarshalText implements the encoding.TextUnmarshaler interface.
-func (k *EncryptingPrivateKey) UnmarshalText(b []byte) error {
-	b, err := decodeWithPrefix(encPrivKeyPrefix, b)
-	if err != nil {
-		return fmt.Errorf("unmarshaling: %w", err)
-	}
-
-	if k.inner, err = x25519.NewPrivateKey(b); err != nil {
-		return fmt.Errorf("converting bytes to private key: %w", err)
-	}
-
-	return nil
-}
-
-func (k EncryptingPrivateKey) String() string {
-	b, err := k.MarshalText()
-	if err != nil {
-		panic(err)
-	}
-	return string(b)
-}

go/nebula/keys.go

@@ -0,0 +1,72 @@
+package nebula
+
+import (
+	"crypto/ed25519"
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+
+	"github.com/slackhq/nebula/cert"
+)
+
+// SigningPrivateKey wraps an ed25519.PrivateKey to provide convenient JSON
+// (un)marshaling methods.
+type SigningPrivateKey ed25519.PrivateKey
+
+// MarshalJSON implements the json.Marshaler interface.
+func (k SigningPrivateKey) MarshalJSON() ([]byte, error) {
+	pemStr := cert.MarshalEd25519PrivateKey(ed25519.PrivateKey(k))
+	return json.Marshal(string(pemStr))
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface.
+func (k *SigningPrivateKey) UnmarshalJSON(b []byte) error {
+	var pemStr string
+	if err := json.Unmarshal(b, &pemStr); err != nil {
+		return fmt.Errorf("unmarshaling into string: %w", err)
+	}
+
+	key, _, err := cert.UnmarshalEd25519PrivateKey([]byte(pemStr))
+	if err != nil {
+		return fmt.Errorf("unmarshaling from PEM: %w", err)
+	}
+
+	*k = SigningPrivateKey(key)
+	return nil
+}
+
+// SigningPublicKey wraps an ed25519.PublicKey to provide convenient JSON
+// (un)marshaling methods.
+type SigningPublicKey ed25519.PublicKey
+
+// MarshalJSON implements the json.Marshaler interface.
+func (k SigningPublicKey) MarshalJSON() ([]byte, error) {
+	pemStr := cert.MarshalEd25519PublicKey(ed25519.PublicKey(k))
+	return json.Marshal(string(pemStr))
+}
+
+// MarshalJSON implements the json.Unmarshaler interface.
+func (k *SigningPublicKey) UnmarshalJSON(b []byte) error {
+	var pemStr string
+	if err := json.Unmarshal(b, &pemStr); err != nil {
+		return fmt.Errorf("unmarshaling into string: %w", err)
+	}
+
+	key, _, err := cert.UnmarshalEd25519PublicKey([]byte(pemStr))
+	if err != nil {
+		return fmt.Errorf("unmarshaling from PEM: %w", err)
+	}
+
+	*k = SigningPublicKey(key)
+	return nil
+}
+
+// GenerateSigningPair generates and returns a new key pair which can be used
+// for signing arbitrary blobs of bytes.
+func GenerateSigningPair() (SigningPublicKey, SigningPrivateKey) {
+	pub, priv, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		panic(fmt.Errorf("generating ed25519 key: %w", err))
+	}
+	return SigningPublicKey(pub), SigningPrivateKey(priv)
+}

go/nebula/nebula.go

@@ -3,32 +3,35 @@
 package nebula
 
 import (
+	"crypto/rand"
 	"fmt"
+	"io"
 	"net"
 	"time"
 
 	"github.com/slackhq/nebula/cert"
+	"golang.org/x/crypto/curve25519"
 )
 
 // HostPublicCredentials contains certificate and signing public keys which are
 // able to be broadcast publicly.
 type HostPublicCredentials struct {
-	Cert       cert.NebulaCertificate
+	CertPEM    string
 	SigningKey SigningPublicKey
 }
 
 // HostPrivateCredentials contains the private key files which will
 // need to be present on a particular host.
 type HostPrivateCredentials struct {
-	EncryptingPrivateKey EncryptingPrivateKey
-	SigningPrivateKey    SigningPrivateKey
+	PrivateKeyPEM     string
+	SigningPrivateKey SigningPrivateKey
 }
 
 // CAPublicCredentials contains certificate and signing public keys which are
 // able to be broadcast publicly. The signing public key is the same one which
 // is embedded into the certificate.
 type CAPublicCredentials struct {
-	Cert       cert.NebulaCertificate
+	CertPEM    string
 	SigningKey SigningPublicKey
 }
 
@@ -39,28 +42,33 @@ type CACredentials struct {
 	SigningPrivateKey SigningPrivateKey
 }
 
-// NewHostCert generates and signs a new host certificate containing the given
-// public key.
-func NewHostCert(
-	caCreds CACredentials,
-	hostPub EncryptingPublicKey,
-	hostName string,
-	ip net.IP,
+// NewHostCertPEM generates and signs a new host certificate containing the
+// given public key.
+func NewHostCertPEM(
+	caCreds CACredentials, hostPubPEM string, hostName string, ip net.IP,
 ) (
-	cert.NebulaCertificate, error,
+	string, error,
 ) {
-	caCert := caCreds.Public.Cert
+	hostPub, _, err := cert.UnmarshalX25519PublicKey([]byte(hostPubPEM))
+	if err != nil {
+		return "", fmt.Errorf("unmarshaling public key PEM: %w", err)
+	}
+
+	caCert, _, err := cert.UnmarshalNebulaCertificateFromPEM([]byte(caCreds.Public.CertPEM))
+	if err != nil {
+		return "", fmt.Errorf("unmarshaling ca.crt: %w", err)
+	}
 
 	issuer, err := caCert.Sha256Sum()
 	if err != nil {
-		return cert.NebulaCertificate{}, fmt.Errorf("getting ca.crt issuer: %w", err)
+		return "", fmt.Errorf("getting ca.crt issuer: %w", err)
 	}
 
 	expireAt := caCert.Details.NotAfter.Add(-1 * time.Second)
 
 	subnet := caCert.Details.Subnets[0]
 	if !subnet.Contains(ip) {
-		return cert.NebulaCertificate{}, fmt.Errorf("invalid ip %q, not contained by network subnet %q", ip, subnet)
+		return "", fmt.Errorf("invalid ip %q, not contained by network subnet %q", ip, subnet)
 	}
 
 	hostCert := cert.NebulaCertificate{
@@ -72,21 +80,26 @@ func NewHostCert(
 			}},
 			NotBefore: time.Now(),
 			NotAfter:  expireAt,
-			PublicKey: hostPub.Bytes(),
+			PublicKey: hostPub,
 			IsCA:      false,
 			Issuer:    issuer,
 		},
 	}
 
-	if err := hostCert.CheckRootConstrains(&caCert); err != nil {
-		return cert.NebulaCertificate{}, fmt.Errorf("validating certificate constraints: %w", err)
+	if err := hostCert.CheckRootConstrains(caCert); err != nil {
+		return "", fmt.Errorf("validating certificate constraints: %w", err)
 	}
 
 	if err := signCert(&hostCert, caCreds.SigningPrivateKey); err != nil {
-		return cert.NebulaCertificate{}, fmt.Errorf("signing host cert with ca.key: %w", err)
+		return "", fmt.Errorf("signing host cert with ca.key: %w", err)
 	}
 
-	return hostCert, nil
+	hostCertPEM, err := hostCert.MarshalToPEM()
+	if err != nil {
+		return "", fmt.Errorf("marshalling host.crt: %w", err)
+	}
+
+	return string(hostCertPEM), nil
 }
 
 // NewHostCredentials generates a new key/cert for a nebula host using the CA
@@ -97,27 +110,39 @@ func NewHostCredentials(
 	pub HostPublicCredentials, priv HostPrivateCredentials, err error,
 ) {
 
-	var (
-		encPrivKey = NewEncryptingPrivateKey()
-		encPubKey  = encPrivKey.PublicKey()
+	// The logic here is largely based on
+	// https://github.com/slackhq/nebula/blob/v1.4.0/cmd/nebula-cert/sign.go
 
-		signingPubKey, signingPrivKey = GenerateSigningPair()
-	)
+	var hostPub, hostKey []byte
+	{
+		var pubkey, privkey [32]byte
+		if _, err = io.ReadFull(rand.Reader, privkey[:]); err != nil {
+			err = fmt.Errorf("reading random bytes to form private key: %w", err)
+			return
+		}
+		curve25519.ScalarBaseMult(&pubkey, &privkey)
+		hostPub, hostKey = pubkey[:], privkey[:]
+	}
 
-	hostCert, err := NewHostCert(caCreds, encPubKey, hostName, ip)
+	signingPubKey, signingPrivKey := GenerateSigningPair()
+
+	hostPubPEM := cert.MarshalX25519PublicKey(hostPub)
+	hostKeyPEM := cert.MarshalX25519PrivateKey(hostKey)
+
+	hostCertPEM, err := NewHostCertPEM(caCreds, string(hostPubPEM), hostName, ip)
 	if err != nil {
 		err = fmt.Errorf("creating host certificate: %w", err)
 		return
 	}
 
 	pub = HostPublicCredentials{
-		Cert:       hostCert,
+		CertPEM:    hostCertPEM,
 		SigningKey: signingPubKey,
 	}
 
 	priv = HostPrivateCredentials{
-		EncryptingPrivateKey: encPrivKey,
-		SigningPrivateKey:    signingPrivKey,
+		PrivateKeyPEM:     string(hostKeyPEM),
+		SigningPrivateKey: signingPrivKey,
 	}
 
 	return
@@ -126,11 +151,14 @@ func NewHostCredentials(
 // NewCACredentials generates a CACredentials. The domain should be the network's root domain,
 // and is included in the signing certificate's Name field.
 func NewCACredentials(domain string, subnet *net.IPNet) (CACredentials, error) {
-	var (
-		signingPubKey, signingPrivKey = GenerateSigningPair()
-		now                           = time.Now()
-		expireAt                      = now.Add(2 * 365 * 24 * time.Hour)
-	)
+
+	// The logic here is largely based on
+	// https://github.com/slackhq/nebula/blob/v1.4.0/cmd/nebula-cert/ca.go
+
+	signingPubKey, signingPrivKey := GenerateSigningPair()
+
+	now := time.Now()
+	expireAt := now.Add(2 * 365 * 24 * time.Hour)
 
 	caCert := cert.NebulaCertificate{
 		Details: cert.NebulaCertificateDetails{
@@ -147,11 +175,33 @@ func NewCACredentials(domain string, subnet *net.IPNet) (CACredentials, error) {
 		return CACredentials{}, fmt.Errorf("signing caCert: %w", err)
 	}
 
+	certPEM, err := caCert.MarshalToPEM()
+	if err != nil {
+		return CACredentials{}, fmt.Errorf("marshaling caCert: %w", err)
+	}
+
 	return CACredentials{
 		Public: CAPublicCredentials{
-			Cert:       caCert,
+			CertPEM:    string(certPEM),
 			SigningKey: signingPubKey,
 		},
 		SigningPrivateKey: signingPrivKey,
 	}, nil
 }
+
+// IPFromHostCertPEM is a convenience function for parsing the IP of a host out
+// of its nebula cert.
+func IPFromHostCertPEM(hostCertPEM string) (net.IP, error) {
+
+	hostCert, _, err := cert.UnmarshalNebulaCertificateFromPEM([]byte(hostCertPEM))
+	if err != nil {
+		return nil, fmt.Errorf("unmarshaling host certificate as PEM: %w", err)
+	}
+
+	ips := hostCert.Details.Ips
+	if len(ips) == 0 {
+		return nil, fmt.Errorf("malformed nebula host cert: no IPs")
+	}
+
+	return ips[0].IP, nil
+}

go/nebula/signing_key.go

@@ -1,78 +0,0 @@
-package nebula
-
-import (
-	"crypto/ed25519"
-	"crypto/rand"
-	"fmt"
-)
-
-var (
-	sigPrivKeyPrefix = []byte("s0")
-	sigPubKeyPrefix  = []byte("S0")
-)
-
-// SigningPrivateKey wraps an ed25519.PrivateKey to provide convenient text
-// (un)marshaling methods.
-type SigningPrivateKey ed25519.PrivateKey
-
-// MarshalText implements the encoding.TextMarshaler interface.
-func (k SigningPrivateKey) MarshalText() ([]byte, error) {
-	return encodeWithPrefix(sigPrivKeyPrefix, k), nil
-}
-
-// UnmarshalText implements the encoding.TextUnmarshaler interface.
-func (k *SigningPrivateKey) UnmarshalText(b []byte) error {
-	b, err := decodeWithPrefix(sigPrivKeyPrefix, b)
-	if err != nil {
-		return fmt.Errorf("unmarshaling: %w", err)
-	}
-
-	*k = SigningPrivateKey(b)
-	return nil
-}
-
-func (k SigningPrivateKey) String() string {
-	b, err := k.MarshalText()
-	if err != nil {
-		panic(err)
-	}
-	return string(b)
-}
-
-// SigningPublicKey wraps an ed25519.PublicKey to provide convenient text
-// (un)marshaling methods.
-type SigningPublicKey ed25519.PublicKey
-
-// MarshalText implements the encoding.TextMarshaler interface.
-func (pk SigningPublicKey) MarshalText() ([]byte, error) {
-	return encodeWithPrefix(sigPubKeyPrefix, pk), nil
-}
-
-// UnmarshalText implements the encoding.TextUnmarshaler interface.
-func (pk *SigningPublicKey) UnmarshalText(b []byte) error {
-	b, err := decodeWithPrefix(sigPubKeyPrefix, b)
-	if err != nil {
-		return fmt.Errorf("unmarshaling: %w", err)
-	}
-
-	*pk = SigningPublicKey(b)
-	return nil
-}
-
-func (pk SigningPublicKey) String() string {
-	b, err := pk.MarshalText()
-	if err != nil {
-		panic(err)
-	}
-	return string(b)
-}
-
-// GenerateSigningPair generates and returns a new key pair which can be used
-// for signing arbitrary blobs of bytes.
-func GenerateSigningPair() (SigningPublicKey, SigningPrivateKey) {
-	pub, priv, err := ed25519.GenerateKey(rand.Reader)
-	if err != nil {
-		panic(fmt.Errorf("generating ed25519 key: %w", err))
-	}
-	return SigningPublicKey(pub), SigningPrivateKey(priv)
-}

go/nebula/util.go

@@ -1,25 +0,0 @@
-package nebula
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-
-	"github.com/jxskiss/base62"
-)
-
-func encodeWithPrefix(prefix []byte, b []byte) []byte {
-	res := make([]byte, 0, len(prefix)+len(b)*4)
-	res = append(res, prefix...)
-	res = base62.EncodeToBuf(res, b)
-	return res
-}
-
-func decodeWithPrefix(prefix []byte, b []byte) ([]byte, error) {
-	if len(b) < len(prefix) {
-		return nil, errors.New("input is too short")
-	} else if !bytes.HasPrefix(b, prefix) {
-		return nil, fmt.Errorf("missing expected prefix %q", prefix)
-	}
-	return base62.Decode(b[len(prefix):])
-}

nix/garage.nix

@@ -1,15 +1,20 @@
 rec {
 
-  version = "1.0.0";
+  version = "0.8.1";
 
   src = builtins.fetchGit {
     name = "garage-v${version}";
     url = "https://git.deuxfleurs.fr/Deuxfleurs/garage.git";
-    rev = "ff093ddbb8485409f389abe7b5e569cb38d222d2";
+    rev = "76230f20282e73a5a5afa33af68152acaf732cf5";
   };
 
+  # TODO compiling garage broke using 24.05, so for now use the pkgs pinned in
+  # the garage repo. Probably will revisit this when garage gets upgraded
+  # anyway.
+  pkgsSrc = (import "${src}/nix/common.nix").pkgsSrc;
+
   package = {
-    pkgsSrc,
+    #pkgsSrc,
     buildSystem,
     hostSystem,
   }: let
@@ -24,16 +29,6 @@ rec {
 
       release = true;
       git_version = version;
-
-      # subset of the default release features, as defined in:
-      # https://git.deuxfleurs.fr/Deuxfleurs/garage/src/commit/ff093ddbb8485409f389abe7b5e569cb38d222d2/nix/compile.nix#L171
-      features = [
-        "garage/bundled-libs"
-        "garage/lmdb"
-        "garage/sqlite"
-        "garage/metrics"
-        "garage/syslog"
-      ];
     };
 
   in

nix/pkgs.nix

@@ -38,9 +38,7 @@ rec {
   supportedSystems = [
     "x86_64-linux"
     "aarch64-linux"
-
-    # rpi, TODO remember why this is disabled, try to re-enable it
-    #"armv7l-linux-musl"
+    #"armv7l-linux-musl" # rpi, I think?
     "i686-linux"
   ];
 

tests/entrypoint.sh

@@ -68,15 +68,14 @@ for file in $test_files; do
     if [ -z "$VERBOSE" ]; then
         output="$TMPDIR/$file.log"
         mkdir -p "$(dirname "$output")"
-        exec 3>"$output"
     else
-        exec 3>&1
+        output=/dev/stdout
     fi
 
     (
         export TEST_CASE_FILE="$file"
 
-        if ! $SHELL -e -x "$root/cases/$file" >&3 2>&1; then
+        if ! $SHELL -e -x "$root/cases/$file" >"$output" 2>&1; then
             echo "$file FAILED"
             if [ -z "$VERBOSE" ]; then
                 echo "output of test is as follows"

tests/utils/with-1-data-1-empty-node-cluster.sh

@@ -40,13 +40,13 @@ if [ ! -d "$XDG_RUNTIME_DIR/isle" ]; then
       allocations:
         - data_path: a/data
           meta_path: a/meta
-          capacity: 1
+          capacity: 100
         - data_path: b/data
           meta_path: b/meta
-          capacity: 1
+          capacity: 100
         - data_path: c/data
           meta_path: c/meta
-          capacity: 1
+          capacity: 100
 EOF
 
     echo "Creating 1-data-1-empty network"