Refactor how signing/encryption keys are typed and (un)marshaled

This required switching all garage admin API calls to the new v1
versions, and redoing how the global bucket key is created so it is
created via the "create key" API call.

AppDir/etc/daemon.yml

@@ -69,7 +69,7 @@ storage:
   # the meta directories can be anywhere (ideally on an SSD).
   #
   # Capacity declares how many gigabytes can be stored in each allocation, and
-  # is required. It must be a multiple of 100.
+  # is required.
   #
   # The ports are all _optional_, and will be automatically assigned if they are
   # not specified. If ports any ports are specified then all should be

default.nix

@@ -69,7 +69,7 @@ in rec {
       '';
     };
 
-    vendorHash = "sha256-P1TXG0fG8/6n37LmM5ApYctqoZzJFlvFAO2Zl85SVvk=";
+    vendorHash = "sha256-33gwBj+6x9I/yz0Qf4G8YXRgC/HfwHCedqzrCE4FHHk=";
 
     subPackages = [
       "./cmd/entrypoint"
@@ -90,7 +90,7 @@ in rec {
 
     inherit buildSystem;
     hostSystem = "${hostPlatform.cpu.name}-unknown-${hostPlatform.kernel.name}-musl";
-    #pkgsSrc = pkgsNix.src;
+    pkgsSrc = pkgsNix.src;
 
   };
 

docs/operator/contributing-storage.md

@@ -35,7 +35,7 @@ storage:
       meta_path: /mnt/drive1/isle/meta
       capacity: 1200
 
-    # 100 GB (the minimum) are being shared from drive2
+    # 100 GB are being shared from drive2
     - data_path: /mnt/drive2/isle/data
       meta_path: /mnt/drive2/isle/meta
       capacity: 100

go/bootstrap/bootstrap.go

@@ -30,10 +30,13 @@ func AppDirPath(appDirPath string) string {
 
 // Garage contains parameters needed to connect to and use the garage cluster.
 type Garage struct {
-	// TODO RPCSecret and GlobalBucketS3APICredentials are duplicated here and
-	// in AdminCreationParams, might as well just use them from there
-	RPCSecret                    string
-	AdminToken                   string
+	// TODO this should be part of some new configuration section related to
+	// secrets which may or may not be granted to this host
+	RPCSecret string
+
+	AdminToken string
+
+	// TODO this should be part of admin.CreationParams
 	GlobalBucketS3APICredentials garage.S3APICredentials
 }
 

go/bootstrap/garage.go

@@ -27,17 +27,6 @@ func (b Bootstrap) GaragePeers() []garage.RemotePeer {
 	return peers
 }
 
-// GarageRPCPeerAddrs returns the full RPC peer address for each known garage
-// instance in the network.
-func (b Bootstrap) GarageRPCPeerAddrs() []string {
-	var addrs []string
-	for _, peer := range b.GaragePeers() {
-		addr := peer.RPCPeerAddr()
-		addrs = append(addrs, addr)
-	}
-	return addrs
-}
-
 // ChooseGaragePeer returns a Peer for a garage instance from the network. It
 // will prefer a garage instance on this particular host, if there is one, but
 // will otherwise return a random endpoint.

go/bootstrap/hosts.go

@@ -78,10 +78,5 @@ type Host struct {
 // This assumes that the Host and its data has already been verified against the
 // CA signing key.
 func (h Host) IP() net.IP {
-	ip, err := nebula.IPFromHostCertPEM(h.PublicCredentials.CertPEM)
-	if err != nil {
-		panic(fmt.Errorf("could not parse IP out of cert for host %q: %w", h.Name, err))
-	}
-
-	return ip
+	return h.PublicCredentials.Cert.Details.Ips[0].IP
 }

go/cmd/entrypoint/admin.go

@@ -149,9 +149,8 @@ var subCmdAdminCreateNetwork = subCmd{
 		}
 
 		garageBootstrap := bootstrap.Garage{
-			RPCSecret:                    randStr(32),
-			AdminToken:                   randStr(32),
-			GlobalBucketS3APICredentials: garage.NewS3APICredentials(),
+			RPCSecret:  randStr(32),
+			AdminToken: randStr(32),
 		}
 
 		hostBootstrap, err := bootstrap.New(
@@ -216,7 +215,17 @@ var subCmdAdminCreateNetwork = subCmd{
 		}
 
 		logger.Info(ctx, "initializing garage shared global bucket")
-		err = garageInitializeGlobalBucket(ctx, logger, hostBootstrap, daemonConfig)
+		garageGlobalBucketCreds, err := garageInitializeGlobalBucket(
+			ctx, logger, hostBootstrap, daemonConfig,
+		)
+
+		hostBootstrap.Garage.GlobalBucketS3APICredentials = garageGlobalBucketCreds
+
+		// rewrite the bootstrap now that the global bucket creds have been
+		// added to it.
+		if err := writeBootstrapToDataDir(hostBootstrap); err != nil {
+			return fmt.Errorf("writing bootstrap file: %w", err)
+		}
 
 		if cErr := (garage.AdminClientError{}); errors.As(err, &cErr) && cErr.StatusCode == 409 {
 			return fmt.Errorf("shared global bucket has already been created, are the storage allocations from a previously initialized isle being used?")
@@ -372,13 +381,23 @@ var subCmdAdminCreateNebulaCert = subCmd{
 			return fmt.Errorf("reading public key from %q: %w", *pubKeyPath, err)
 		}
 
-		nebulaHostCertPEM, err := nebula.NewHostCertPEM(
-			adm.Nebula.CACredentials, string(hostPubPEM), *hostName, ip,
+		var hostPub nebula.EncryptingPublicKey
+		if err := hostPub.UnmarshalNebulaPEM(hostPubPEM); err != nil {
+			return fmt.Errorf("unmarshaling public key as PEM: %w", err)
+		}
+
+		nebulaHostCert, err := nebula.NewHostCert(
+			adm.Nebula.CACredentials, hostPub, *hostName, ip,
 		)
 		if err != nil {
 			return fmt.Errorf("creating cert: %w", err)
 		}
 
+		nebulaHostCertPEM, err := nebulaHostCert.MarshalToPEM()
+		if err != nil {
+			return fmt.Errorf("marshaling cert to PEM: %w", err)
+		}
+
 		if _, err := os.Stdout.Write([]byte(nebulaHostCertPEM)); err != nil {
 			return fmt.Errorf("writing to stdout: %w", err)
 		}

go/cmd/entrypoint/daemon_util.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"isle/bootstrap"
 	"isle/daemon"
-	"isle/garage"
+	"isle/garage/garagesrv"
 	"time"
 )
 
@@ -29,7 +29,7 @@ func coalesceDaemonConfigAndBootstrap(
 
 		for i, alloc := range allocs {
 
-			id, rpcPort, err := garage.InitAlloc(alloc.MetaPath, alloc.RPCPort)
+			id, rpcPort, err := garagesrv.InitAlloc(alloc.MetaPath, alloc.RPCPort)
 			if err != nil {
 				return bootstrap.Bootstrap{}, daemon.Config{}, fmt.Errorf("initializing alloc at %q: %w", alloc.MetaPath, err)
 			}

go/cmd/entrypoint/garage_util.go

@@ -2,10 +2,11 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"isle/bootstrap"
 	"isle/daemon"
 	"isle/garage"
-	"fmt"
+	"isle/garage/garagesrv"
 	"net"
 	"path/filepath"
 	"strconv"
@@ -30,12 +31,12 @@ func newGarageAdminClient(
 	thisHost := hostBootstrap.ThisHost()
 
 	return garage.NewAdminClient(
+		garageAdminClientLogger(logger),
 		net.JoinHostPort(
 			thisHost.IP().String(),
 			strconv.Itoa(daemonConfig.Storage.Allocations[0].AdminPort),
 		),
 		hostBootstrap.Garage.AdminToken,
-		garageAdminClientLogger(logger),
 	)
 }
 
@@ -68,9 +69,9 @@ func waitForGarageAndNebula(
 		)
 
 		adminClient := garage.NewAdminClient(
+			adminClientLogger,
 			adminAddr,
 			hostBootstrap.Garage.AdminToken,
-			adminClientLogger,
 		)
 
 		ctx := mctx.Annotate(ctx, "garageAdminAddr", adminAddr)
@@ -128,18 +129,15 @@ func garageWriteChildConfig(
 		envRuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
 	)
 
-	err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
+	err := garagesrv.WriteGarageTomlFile(garageTomlPath, garagesrv.GarageTomlData{
 		MetaPath: alloc.MetaPath,
 		DataPath: alloc.DataPath,
 
 		RPCSecret:  hostBootstrap.Garage.RPCSecret,
 		AdminToken: hostBootstrap.Garage.AdminToken,
 
-		RPCAddr:   peer.RPCAddr(),
-		S3APIAddr: peer.S3APIAddr(),
-		AdminAddr: peer.AdminAddr(),
-
-		BootstrapPeers: hostBootstrap.GarageRPCPeerAddrs(),
+		LocalPeer:      peer,
+		BootstrapPeers: hostBootstrap.GaragePeers(),
 	})
 
 	if err != nil {
@@ -184,62 +182,36 @@ func garageInitializeGlobalBucket(
 	logger *mlog.Logger,
 	hostBootstrap bootstrap.Bootstrap,
 	daemonConfig daemon.Config,
-) error {
+) (
+	garage.S3APICredentials, error,
+) {
+	adminClient := newGarageAdminClient(logger, hostBootstrap, daemonConfig)
 
-	var (
-		adminClient       = newGarageAdminClient(logger, hostBootstrap, daemonConfig)
-		globalBucketCreds = hostBootstrap.Garage.GlobalBucketS3APICredentials
+	creds, err := adminClient.CreateS3APICredentials(
+		ctx, garage.GlobalBucketS3APICredentialsName,
 	)
-
-	// first attempt to import the key
-	err := adminClient.Do(ctx, nil, "POST", "/v0/key/import", map[string]string{
-		"accessKeyId":     globalBucketCreds.ID,
-		"secretAccessKey": globalBucketCreds.Secret,
-		"name":            "shared-global-bucket-key",
-	})
-
 	if err != nil {
-		return fmt.Errorf("importing global bucket key into garage: %w", err)
+		return creds, fmt.Errorf("creating global bucket credentials: %w", err)
 	}
 
-	// create global bucket
-	err = adminClient.Do(ctx, nil, "POST", "/v0/bucket", map[string]string{
-		"globalAlias": garage.GlobalBucket,
-	})
-
+	bucketID, err := adminClient.CreateBucket(ctx, garage.GlobalBucket)
 	if err != nil {
-		return fmt.Errorf("creating global bucket: %w", err)
+		return creds, fmt.Errorf("creating global bucket: %w", err)
 	}
 
-	// retrieve newly created bucket's id
-	var getBucketRes struct {
-		ID string `json:"id"`
+	if err := adminClient.GrantBucketPermissions(
+		ctx,
+		bucketID,
+		creds.ID,
+		garage.BucketPermissionRead,
+		garage.BucketPermissionWrite,
+	); err != nil {
+		return creds, fmt.Errorf(
+			"granting permissions to shared global bucket key: %w", err,
+		)
 	}
 
-	err = adminClient.Do(
-		ctx, &getBucketRes,
-		"GET", "/v0/bucket?globalAlias="+garage.GlobalBucket, nil,
-	)
-
-	if err != nil {
-		return fmt.Errorf("fetching global bucket id: %w", err)
-	}
-
-	// allow shared global bucket key to perform all operations
-	err = adminClient.Do(ctx, nil, "POST", "/v0/bucket/allow", map[string]interface{}{
-		"bucketId":    getBucketRes.ID,
-		"accessKeyId": globalBucketCreds.ID,
-		"permissions": map[string]bool{
-			"read":  true,
-			"write": true,
-		},
-	})
-
-	if err != nil {
-		return fmt.Errorf("granting permissions to shared global bucket key: %w", err)
-	}
-
-	return nil
+	return creds, nil
 }
 
 func garageApplyLayout(
@@ -254,62 +226,25 @@ func garageApplyLayout(
 		thisHost    = hostBootstrap.ThisHost()
 		hostName    = thisHost.Name
 		allocs      = daemonConfig.Storage.Allocations
+		peers       = make([]garage.PeerLayout, len(allocs))
 	)
 
-	type peerLayout struct {
-		Capacity int      `json:"capacity"`
-		Zone     string   `json:"zone"`
-		Tags     []string `json:"tags"`
-	}
+	for i, alloc := range allocs {
 
-	{
-		clusterLayout := map[string]peerLayout{}
+		id := bootstrapGarageHostForAlloc(thisHost, alloc).ID
 
-		for _, alloc := range allocs {
-
-			id := bootstrapGarageHostForAlloc(thisHost, alloc).ID
-
-			zone := hostName
-			if alloc.Zone != "" {
-				zone = alloc.Zone
-			}
-
-			clusterLayout[id] = peerLayout{
-				Capacity: alloc.Capacity,
-				Zone:     zone,
-				Tags:     []string{},
-			}
+		zone := hostName
+		if alloc.Zone != "" {
+			zone = alloc.Zone
 		}
 
-		err := adminClient.Do(ctx, nil, "POST", "/v0/layout", clusterLayout)
-		if err != nil {
-			return fmt.Errorf("staging layout changes: %w", err)
+		peers[i] = garage.PeerLayout{
+			ID:       id,
+			Capacity: alloc.Capacity * 1_000_000_000,
+			Zone:     zone,
+			Tags:     []string{},
 		}
 	}
 
-	var clusterLayout struct {
-		Version           int                   `json:"version"`
-		StagedRoleChanges map[string]peerLayout `json:"stagedRoleChanges"`
-	}
-
-	if err := adminClient.Do(ctx, &clusterLayout, "GET", "/v0/layout", nil); err != nil {
-		return fmt.Errorf("retrieving staged layout change: %w", err)
-	}
-
-	if len(clusterLayout.StagedRoleChanges) == 0 {
-		return nil
-	}
-
-	applyClusterLayout := struct {
-		Version int `json:"version"`
-	}{
-		Version: clusterLayout.Version + 1,
-	}
-
-	err := adminClient.Do(ctx, nil, "POST", "/v0/layout/apply", applyClusterLayout)
-	if err != nil {
-		return fmt.Errorf("applying new layout (new version:%d): %w", applyClusterLayout.Version, err)
-	}
-
-	return nil
+	return adminClient.ApplyLayout(ctx, peers)
 }

go/cmd/entrypoint/nebula.go

@@ -4,8 +4,6 @@ import (
 	"fmt"
 	"isle/jsonutil"
 	"os"
-
-	"github.com/slackhq/nebula/cert"
 )
 
 var subCmdNebulaShow = subCmd{
@@ -23,10 +21,10 @@ var subCmdNebulaShow = subCmd{
 			return fmt.Errorf("loading host bootstrap: %w", err)
 		}
 
-		caPublicCreds := hostBootstrap.CAPublicCredentials
-		caCert, _, err := cert.UnmarshalNebulaCertificateFromPEM([]byte(caPublicCreds.CertPEM))
+		caCert := hostBootstrap.CAPublicCredentials.Cert
+		caCertPEM, err := caCert.MarshalToPEM()
 		if err != nil {
-			return fmt.Errorf("unmarshaling ca.crt: %w", err)
+			return fmt.Errorf("marshaling CA cert to PEM: %w", err)
 		}
 
 		if len(caCert.Details.Subnets) != 1 {
@@ -48,7 +46,7 @@ var subCmdNebulaShow = subCmd{
 			SubnetCIDR  string
 			Lighthouses []outLighthouse
 		}{
-			CACert:     caPublicCreds.CertPEM,
+			CACert:     string(caCertPEM),
 			SubnetCIDR: subnet.String(),
 		}
 

go/cmd/entrypoint/nebula_util.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 
 	"code.betamike.com/micropelago/pmux/pmuxlib"
+	"github.com/slackhq/nebula/cert"
 )
 
 // waitForNebula waits for the nebula interface to have been started up. It does
@@ -56,11 +57,29 @@ func nebulaPmuxProcConfig(
 		staticHostMap[ip] = []string{host.Nebula.PublicAddr}
 	}
 
+	caCertPEM, err := hostBootstrap.CAPublicCredentials.Cert.MarshalToPEM()
+	if err != nil {
+		return pmuxlib.ProcessConfig{}, fmt.Errorf(
+			"marshaling CA cert to PEM: :%w", err,
+		)
+	}
+
+	hostCertPEM, err := hostBootstrap.PublicCredentials.Cert.MarshalToPEM()
+	if err != nil {
+		return pmuxlib.ProcessConfig{}, fmt.Errorf(
+			"marshaling host cert to PEM: :%w", err,
+		)
+	}
+
+	hostKeyPEM := cert.MarshalX25519PrivateKey(
+		hostBootstrap.PrivateCredentials.EncryptingPrivateKey.Bytes(),
+	)
+
 	config := map[string]interface{}{
 		"pki": map[string]string{
-			"ca":   hostBootstrap.CAPublicCredentials.CertPEM,
-			"cert": hostBootstrap.PublicCredentials.CertPEM,
-			"key":  hostBootstrap.PrivateCredentials.PrivateKeyPEM,
+			"ca":   string(caCertPEM),
+			"cert": string(hostCertPEM),
+			"key":  string(hostKeyPEM),
 		},
 		"static_host_map": staticHostMap,
 		"punchy": map[string]bool{

go/garage/admin_client.go

@@ -14,7 +14,22 @@ import (
 	"github.com/mediocregopher/mediocre-go-lib/v2/mlog"
 )
 
-// AdminClientError gets returned from AdminClient's Do method for non-200
+// BucketID is a unique identifier for a bucket in garage. It is different than
+// the bucket's alias, since a bucket may have multiple aliases.
+type BucketID string
+
+// BucketPermission describes a permission an S3APICredentials may have when
+// interacting with a bucket.
+type BucketPermission string
+
+// Enumeration of BucketPermissions.
+const (
+	BucketPermissionRead  BucketPermission = "read"
+	BucketPermissionWrite BucketPermission = "write"
+	BucketPermissionOwner BucketPermission = "owner"
+)
+
+// AdminClientError gets returned from AdminClient Do methods for non-200
 // errors.
 type AdminClientError struct {
 	StatusCode int
@@ -28,32 +43,32 @@ func (e AdminClientError) Error() string {
 // AdminClient is a helper type for performing actions against the garage admin
 // interface.
 type AdminClient struct {
+	logger     *mlog.Logger
 	c          *http.Client
 	addr       string
 	adminToken string
-	logger     *mlog.Logger
 }
 
 // NewAdminClient initializes and returns an AdminClient which will use the
 // given address and adminToken for all requests made.
 //
 // If Logger is nil then logs will be suppressed.
-func NewAdminClient(addr, adminToken string, logger *mlog.Logger) *AdminClient {
+func NewAdminClient(logger *mlog.Logger, addr, adminToken string) *AdminClient {
 	return &AdminClient{
+		logger: logger,
 		c: &http.Client{
 			Transport: http.DefaultTransport.(*http.Transport).Clone(),
 		},
 		addr:       addr,
 		adminToken: adminToken,
-		logger:     logger,
 	}
 }
 
-// Do performs an HTTP request with the given method (GET, POST) and path, and
+// do performs an HTTP request with the given method (GET, POST) and path, and
 // using the json marshaling of the given body as the request body (unless body
 // is nil). It will JSON unmarshal the response into rcv, unless rcv is nil.
-func (c *AdminClient) Do(
-	ctx context.Context, rcv interface{}, method, path string, body interface{},
+func (c *AdminClient) do(
+	ctx context.Context, rcv any, method, path string, body any,
 ) error {
 
 	var bodyR io.Reader
@@ -112,7 +127,6 @@ func (c *AdminClient) Do(
 	}
 
 	if rcv == nil {
-
 		if _, err := io.Copy(io.Discard, res.Body); err != nil {
 			return fmt.Errorf("discarding response body: %w", err)
 		}
@@ -138,13 +152,14 @@ func (c *AdminClient) Wait(ctx context.Context) error {
 			time.Sleep(250 * time.Millisecond)
 		}
 
+		// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Nodes/operation/GetNodes
 		var clusterStatus struct {
-			KnownNodes map[string]struct {
-				IsUp bool `json:"is_up"`
-			} `json:"knownNodes"`
+			Nodes []struct {
+				IsUp bool `json:"isUp"`
+			} `json:"nodes"`
 		}
 
-		err := c.Do(ctx, &clusterStatus, "GET", "/v0/status", nil)
+		err := c.do(ctx, &clusterStatus, "GET", "/v1/status", nil)
 
 		if ctxErr := ctx.Err(); ctxErr != nil {
 			return ctxErr
@@ -156,14 +171,14 @@ func (c *AdminClient) Wait(ctx context.Context) error {
 
 		var numUp int
 
-		for _, knownNode := range clusterStatus.KnownNodes {
-			if knownNode.IsUp {
+		for _, node := range clusterStatus.Nodes {
+			if node.IsUp {
 				numUp++
 			}
 		}
 
 		ctx := mctx.Annotate(ctx,
-			"numKnownNodes", len(clusterStatus.KnownNodes),
+			"numNodes", len(clusterStatus.Nodes),
 			"numUp", numUp,
 		)
 
@@ -175,3 +190,134 @@ func (c *AdminClient) Wait(ctx context.Context) error {
 		c.logger.Debug(ctx, "instance not online yet, will continue waiting")
 	}
 }
+
+// CreateS3APICredentials creates an S3APICredentials with the given name. The
+// credentials must be associated with a bucket via a call to
+// GrantBucketPermissions in order to be useful.
+func (c *AdminClient) CreateS3APICredentials(
+	ctx context.Context, name string,
+) (
+	S3APICredentials, error,
+) {
+	// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Key/operation/AddKey
+
+	var res struct {
+		ID     string `json:"accessKeyId"`
+		Secret string `json:"secretAccessKey"`
+	}
+
+	// first attempt to import the key
+	err := c.do(
+		ctx, &res, "POST", "/v1/key", map[string]string{"name": name},
+	)
+
+	return S3APICredentials{
+		ID:     res.ID,
+		Secret: res.Secret,
+	}, err
+}
+
+// CreateBucket creates a bucket with the given global alias, returning its ID.
+func (c *AdminClient) CreateBucket(
+	ctx context.Context, globalAlias string,
+) (
+	BucketID, error,
+) {
+	// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Bucket/operation/PutBucketGlobalAlias
+
+	var res struct {
+		ID string `json:"id"`
+	}
+
+	err := c.do(
+		ctx, &res, "POST", "/v1/bucket", map[string]string{
+			"globalAlias": globalAlias,
+		},
+	)
+
+	return BucketID(res.ID), err
+}
+
+// GrantBucketPermissions grants the S3APICredentials with the given ID
+// permission(s) to interact with the bucket of the given ID.
+func (c *AdminClient) GrantBucketPermissions(
+	ctx context.Context,
+	bucketID BucketID,
+	credentialsID string,
+	perms ...BucketPermission,
+) error {
+	permsMap := map[BucketPermission]bool{}
+	for _, perm := range perms {
+		permsMap[perm] = true
+	}
+
+	// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Bucket/operation/AllowBucketKey
+	return c.do(ctx, nil, "POST", "/v1/bucket/allow", map[string]any{
+		"bucketId":    bucketID,
+		"accessKeyId": credentialsID,
+		"permissions": permsMap,
+	})
+}
+
+// PeerLayout describes the properties of a garage peer in the context of the
+// layout of the cluster.
+type PeerLayout struct {
+	ID       string
+	Capacity int // Gb (SI units)
+	Zone     string
+	Tags     []string
+}
+
+// ApplyLayout modifies the layout of the garage cluster. Only layout of the
+// given peers will be modified/created, other peers are not affected.
+func (c *AdminClient) ApplyLayout(
+	ctx context.Context, peers []PeerLayout,
+) error {
+	type peerLayout struct {
+		ID       string   `json:"id"`
+		Capacity int      `json:"capacity"`
+		Zone     string   `json:"zone"`
+		Tags     []string `json:"tags"`
+	}
+
+	{
+		// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Layout/operation/ApplyLayout
+		clusterLayout := make([]peerLayout, len(peers))
+		for i := range peers {
+			clusterLayout[i] = peerLayout(peers[i])
+		}
+
+		err := c.do(ctx, nil, "POST", "/v1/layout", clusterLayout)
+		if err != nil {
+			return fmt.Errorf("staging layout changes: %w", err)
+		}
+	}
+
+	// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Layout/operation/GetLayout
+	var clusterLayout struct {
+		Version           int          `json:"version"`
+		StagedRoleChanges []peerLayout `json:"stagedRoleChanges"`
+	}
+
+	if err := c.do(ctx, &clusterLayout, "GET", "/v1/layout", nil); err != nil {
+		return fmt.Errorf("retrieving staged layout change: %w", err)
+	}
+
+	if len(clusterLayout.StagedRoleChanges) == 0 {
+		return nil
+	}
+
+	// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Layout/operation/ApplyLayout
+	applyClusterLayout := struct {
+		Version int `json:"version"`
+	}{
+		Version: clusterLayout.Version + 1,
+	}
+
+	err := c.do(ctx, nil, "POST", "/v1/layout/apply", applyClusterLayout)
+	if err != nil {
+		return fmt.Errorf("applying new layout (new version:%d): %w", applyClusterLayout.Version, err)
+	}
+
+	return nil
+}

go/garage/client.go

@@ -1,8 +1,6 @@
 package garage
 
 import (
-	"crypto/rand"
-	"encoding/hex"
 	"errors"
 	"fmt"
 
@@ -10,14 +8,6 @@ import (
 	"github.com/minio/minio-go/v7/pkg/credentials"
 )
 
-func randStr(l int) string {
-	b := make([]byte, l)
-	if _, err := rand.Read(b); err != nil {
-		panic(err)
-	}
-	return hex.EncodeToString(b)
-}
-
 // IsKeyNotFound returns true if the given error is the result of a key not
 // being found in a bucket.
 func IsKeyNotFound(err error) bool {
@@ -35,14 +25,6 @@ type S3APICredentials struct {
 	Secret string
 }
 
-// NewS3APICredentials returns a new usable instance of S3APICredentials.
-func NewS3APICredentials() S3APICredentials {
-	return S3APICredentials{
-		ID:     randStr(8),
-		Secret: randStr(32),
-	}
-}
-
 // NewS3APIClient returns a minio client configured to use the given garage S3 API
 // endpoint.
 func NewS3APIClient(addr string, creds S3APICredentials) S3APIClient {

go/garage/garage.go

@@ -1,17 +1,7 @@
-// Package garage contains helper functions and types which are useful for
-// setting up garage configs, processes, and deployments.
+// Package garage contains types and helpers related to interacting with garage
+// processes via garage's APIs.
 package garage
 
-import (
-	"encoding/hex"
-	"errors"
-	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"strconv"
-)
-
 const (
 
 	// Region is the region which garage is configured with.
@@ -21,133 +11,9 @@ const (
 	// accessible to all hosts in the network.
 	GlobalBucket = "global-shared"
 
+	GlobalBucketS3APICredentialsName = "global-shared-key"
+
 	// ReplicationFactor indicates the replication factor set on the garage
 	// cluster. We currently only support a factor of 3.
 	ReplicationFactor = 3
 )
-
-func nodeKeyPath(metaDirPath string) string {
-	return filepath.Join(metaDirPath, "node_key")
-}
-
-func nodeKeyPubPath(metaDirPath string) string {
-	return filepath.Join(metaDirPath, "node_key.pub")
-}
-
-func nodeRPCPortPath(metaDirPath string) string {
-	return filepath.Join(metaDirPath, "isle", "rpc_port")
-}
-
-// loadAllocID returns the peer ID (ie the public key) of the node at the given
-// meta directory.
-func loadAllocID(metaDirPath string) (string, error) {
-	nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
-
-	pubKey, err := os.ReadFile(nodeKeyPubPath)
-	if err != nil {
-		return "", fmt.Errorf("reading %q: %w", nodeKeyPubPath, err)
-	}
-
-	return hex.EncodeToString(pubKey), nil
-}
-
-// InitAlloc initializes the meta directory and keys for a particular
-// allocation, if it hasn't been done so already. It returns the peer ID (ie the
-// public key) and the rpc port in any case.
-func InitAlloc(metaDirPath string, initRPCPort int) (string, int, error) {
-
-	initDirFor := func(path string) error {
-		dir := filepath.Dir(path)
-		return os.MkdirAll(dir, 0750)
-	}
-
-	var err error
-
-	exists := func(path string) bool {
-
-		if err != nil {
-			return false
-
-		} else if _, err = os.Stat(path); errors.Is(err, fs.ErrNotExist) {
-			err = nil
-			return false
-
-		} else if err != nil {
-			err = fmt.Errorf("checking if %q exists: %w", path, err)
-			return false
-		}
-
-		return true
-	}
-
-	nodeKeyPath := nodeKeyPath(metaDirPath)
-	nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
-	nodeRPCPortPath := nodeRPCPortPath(metaDirPath)
-
-	nodeKeyPathExists := exists(nodeKeyPath)
-	nodeKeyPubPathExists := exists(nodeKeyPubPath)
-	nodeRPCPortPathExists := exists(nodeRPCPortPath)
-
-	if err != nil {
-		return "", 0, err
-
-	} else if nodeKeyPubPathExists != nodeKeyPathExists {
-		return "", 0, fmt.Errorf("%q or %q exist without the other existing", nodeKeyPath, nodeKeyPubPath)
-
-	}
-
-	var (
-		pubKeyStr string
-		rpcPort   int
-	)
-
-	if nodeKeyPathExists {
-
-		if pubKeyStr, err = loadAllocID(metaDirPath); err != nil {
-			return "", 0, fmt.Errorf("reading node public key file: %w", err)
-		}
-
-	} else {
-
-		if err := initDirFor(nodeKeyPath); err != nil {
-			return "", 0, fmt.Errorf("creating directory for %q: %w", nodeKeyPath, err)
-		}
-
-		pubKey, privKey := GeneratePeerKey()
-
-		if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
-			return "", 0, fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
-
-		} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
-			return "", 0, fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
-		}
-
-		pubKeyStr = hex.EncodeToString(pubKey)
-	}
-
-	if nodeRPCPortPathExists {
-
-		if rpcPortStr, err := os.ReadFile(nodeRPCPortPath); err != nil {
-			return "", 0, fmt.Errorf("reading rpc port from %q: %w", nodeRPCPortPath, err)
-
-		} else if rpcPort, err = strconv.Atoi(string(rpcPortStr)); err != nil {
-			return "", 0, fmt.Errorf("parsing rpc port %q from %q: %w", rpcPortStr, nodeRPCPortPath, err)
-		}
-
-	} else {
-
-		if err := initDirFor(nodeRPCPortPath); err != nil {
-			return "", 0, fmt.Errorf("creating directory for %q: %w", nodeRPCPortPath, err)
-		}
-
-		rpcPortStr := strconv.Itoa(initRPCPort)
-
-		if err := os.WriteFile(nodeRPCPortPath, []byte(rpcPortStr), 0440); err != nil {
-			return "", 0, fmt.Errorf("writing rpc port %q to %q: %w", rpcPortStr, nodeRPCPortPath, err)
-		}
-
-		rpcPort = initRPCPort
-	}
-
-	return pubKeyStr, rpcPort, nil
-}

go/garage/garagesrv/garagesrv.go

@@ -0,0 +1,152 @@
+// Package garage contains helper functions and types which are useful for
+// setting up garage configs, processes, and deployments.
+package garagesrv
+
+import (
+	"crypto/ed25519"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strconv"
+)
+
+func nodeKeyPath(metaDirPath string) string {
+	return filepath.Join(metaDirPath, "node_key")
+}
+
+func nodeKeyPubPath(metaDirPath string) string {
+	return filepath.Join(metaDirPath, "node_key.pub")
+}
+
+func nodeRPCPortPath(metaDirPath string) string {
+	return filepath.Join(metaDirPath, "isle", "rpc_port")
+}
+
+// loadAllocID returns the peer ID (ie the public key) of the node at the given
+// meta directory.
+func loadAllocID(metaDirPath string) (string, error) {
+	nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
+
+	pubKey, err := os.ReadFile(nodeKeyPubPath)
+	if err != nil {
+		return "", fmt.Errorf("reading %q: %w", nodeKeyPubPath, err)
+	}
+
+	return hex.EncodeToString(pubKey), nil
+}
+
+// generatePeerKey generates and returns a public/private key pair for a garage
+// instance.
+func generatePeerKey() (pubKey, privKey []byte) {
+	pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		panic(err)
+	}
+
+	return pubKey, privKey
+}
+
+// InitAlloc initializes the meta directory and keys for a particular
+// allocation, if it hasn't been done so already. It returns the peer ID (ie the
+// public key) and the rpc port in any case.
+func InitAlloc(metaDirPath string, initRPCPort int) (string, int, error) {
+
+	initDirFor := func(path string) error {
+		dir := filepath.Dir(path)
+		return os.MkdirAll(dir, 0750)
+	}
+
+	var err error
+
+	exists := func(path string) bool {
+
+		if err != nil {
+			return false
+
+		} else if _, err = os.Stat(path); errors.Is(err, fs.ErrNotExist) {
+			err = nil
+			return false
+
+		} else if err != nil {
+			err = fmt.Errorf("checking if %q exists: %w", path, err)
+			return false
+		}
+
+		return true
+	}
+
+	nodeKeyPath := nodeKeyPath(metaDirPath)
+	nodeKeyPubPath := nodeKeyPubPath(metaDirPath)
+	nodeRPCPortPath := nodeRPCPortPath(metaDirPath)
+
+	nodeKeyPathExists := exists(nodeKeyPath)
+	nodeKeyPubPathExists := exists(nodeKeyPubPath)
+	nodeRPCPortPathExists := exists(nodeRPCPortPath)
+
+	if err != nil {
+		return "", 0, err
+
+	} else if nodeKeyPubPathExists != nodeKeyPathExists {
+		return "", 0, fmt.Errorf("%q or %q exist without the other existing", nodeKeyPath, nodeKeyPubPath)
+
+	}
+
+	var (
+		pubKeyStr string
+		rpcPort   int
+	)
+
+	if nodeKeyPathExists {
+
+		if pubKeyStr, err = loadAllocID(metaDirPath); err != nil {
+			return "", 0, fmt.Errorf("reading node public key file: %w", err)
+		}
+
+	} else {
+
+		if err := initDirFor(nodeKeyPath); err != nil {
+			return "", 0, fmt.Errorf("creating directory for %q: %w", nodeKeyPath, err)
+		}
+
+		pubKey, privKey := generatePeerKey()
+
+		if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
+			return "", 0, fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
+
+		} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
+			return "", 0, fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
+		}
+
+		pubKeyStr = hex.EncodeToString(pubKey)
+	}
+
+	if nodeRPCPortPathExists {
+
+		if rpcPortStr, err := os.ReadFile(nodeRPCPortPath); err != nil {
+			return "", 0, fmt.Errorf("reading rpc port from %q: %w", nodeRPCPortPath, err)
+
+		} else if rpcPort, err = strconv.Atoi(string(rpcPortStr)); err != nil {
+			return "", 0, fmt.Errorf("parsing rpc port %q from %q: %w", rpcPortStr, nodeRPCPortPath, err)
+		}
+
+	} else {
+
+		if err := initDirFor(nodeRPCPortPath); err != nil {
+			return "", 0, fmt.Errorf("creating directory for %q: %w", nodeRPCPortPath, err)
+		}
+
+		rpcPortStr := strconv.Itoa(initRPCPort)
+
+		if err := os.WriteFile(nodeRPCPortPath, []byte(rpcPortStr), 0440); err != nil {
+			return "", 0, fmt.Errorf("writing rpc port %q to %q: %w", rpcPortStr, nodeRPCPortPath, err)
+		}
+
+		rpcPort = initRPCPort
+	}
+
+	return pubKeyStr, rpcPort, nil
+}

go/garage/garagesrv/tpl.go

@@ -1,10 +1,13 @@
-package garage
+package garagesrv
 
 import (
 	"fmt"
 	"io"
 	"os"
+	"strconv"
 	"text/template"
+
+	"isle/garage"
 )
 
 // GarageTomlData describes all fields needed for rendering a garage.toml
@@ -16,11 +19,8 @@ type GarageTomlData struct {
 	RPCSecret  string
 	AdminToken string
 
-	RPCAddr   string
-	S3APIAddr string
-	AdminAddr string
-
-	BootstrapPeers []string
+	garage.LocalPeer
+	BootstrapPeers []garage.RemotePeer
 }
 
 var garageTomlTpl = template.Must(template.New("").Parse(`
@@ -28,14 +28,14 @@ var garageTomlTpl = template.Must(template.New("").Parse(`
 metadata_dir = "{{ .MetaPath }}"
 data_dir = "{{ .DataPath }}"
 
-replication_mode = "3"
+replication_mode = "` + strconv.Itoa(garage.ReplicationFactor) + `"
 
 rpc_secret = "{{ .RPCSecret }}"
 rpc_bind_addr = "{{ .RPCAddr }}"
 rpc_public_addr = "{{ .RPCAddr }}"
 
 bootstrap_peers = [{{- range .BootstrapPeers }}
-"{{ . }}",
+"{{ .RPCPeerAddr }}",
 {{ end -}}]
 
 [s3_api]

go/garage/peer.go

@@ -1,8 +1,6 @@
 package garage
 
 import (
-	"crypto/ed25519"
-	"crypto/rand"
 	"fmt"
 	"net"
 	"strconv"
@@ -24,17 +22,6 @@ type LocalPeer struct {
 	AdminPort int
 }
 
-// GeneratePeerKey generates and returns a public/private key pair for a garage
-// instance.
-func GeneratePeerKey() (pubKey, privKey []byte) {
-	pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
-	if err != nil {
-		panic(err)
-	}
-
-	return pubKey, privKey
-}
-
 // RPCAddr returns the address of the peer's RPC port.
 func (p RemotePeer) RPCAddr() string {
 	return net.JoinHostPort(p.IP, strconv.Itoa(p.RPCPort))

go/go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/gopherjs/gopherjs v1.17.2 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/jtolds/gls v4.20.0+incompatible // indirect
+	github.com/jxskiss/base62 v1.1.0 // indirect
 	github.com/klauspost/compress v1.13.5 // indirect
 	github.com/klauspost/cpuid v1.3.1 // indirect
 	github.com/minio/md5-simd v1.1.0 // indirect

go/go.sum

@@ -24,6 +24,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw=
+github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc=
 github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4=
 github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=

go/nebula/encrypting_key.go

@@ -0,0 +1,117 @@
+package nebula
+
+import (
+	"crypto/ecdh"
+	"crypto/rand"
+	"fmt"
+
+	"github.com/slackhq/nebula/cert"
+)
+
+var (
+	encPrivKeyPrefix = []byte("x0")
+	encPubKeyPrefix  = []byte("X0")
+
+	x25519 = ecdh.X25519()
+)
+
+// EncryptingPublicKey wraps an X25519-based ECDH public key to provide
+// convenient text (un)marshaling methods.
+type EncryptingPublicKey struct{ inner *ecdh.PublicKey }
+
+// MarshalText implements the encoding.TextMarshaler interface.
+func (pk EncryptingPublicKey) MarshalText() ([]byte, error) {
+	return encodeWithPrefix(encPubKeyPrefix, pk.inner.Bytes()), nil
+}
+
+// Bytes returns the raw bytes of the EncryptingPublicKey.
+func (k EncryptingPublicKey) Bytes() []byte {
+	return k.inner.Bytes()
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface.
+func (pk *EncryptingPublicKey) UnmarshalText(b []byte) error {
+	b, err := decodeWithPrefix(encPubKeyPrefix, b)
+	if err != nil {
+		return fmt.Errorf("unmarshaling: %w", err)
+	}
+
+	if pk.inner, err = x25519.NewPublicKey(b); err != nil {
+		return fmt.Errorf("converting bytes to public key: %w", err)
+	}
+
+	return nil
+}
+
+// UnmarshalNebulaPEM unmarshals the EncryptingPublicKey as a nebula host public
+// key PEM.
+func (pk *EncryptingPublicKey) UnmarshalNebulaPEM(b []byte) error {
+	b, _, err := cert.UnmarshalEd25519PublicKey(b)
+	if err != nil {
+		return fmt.Errorf("unmarshaling: %w", err)
+	}
+
+	if pk.inner, err = x25519.NewPublicKey(b); err != nil {
+		return fmt.Errorf("converting bytes to public key: %w", err)
+	}
+
+	return nil
+}
+
+func (pk EncryptingPublicKey) String() string {
+	b, err := pk.MarshalText()
+	if err != nil {
+		panic(err)
+	}
+	return string(b)
+}
+
+// EncryptingPrivateKey wraps an X25519-based ECDH private key to provide
+// convenient text (un)marshaling methods.
+type EncryptingPrivateKey struct{ inner *ecdh.PrivateKey }
+
+// NewEncryptingPrivateKey generates and returns a fresh EncryptingPrivateKey.
+func NewEncryptingPrivateKey() EncryptingPrivateKey {
+	k, err := x25519.GenerateKey(rand.Reader)
+	if err != nil {
+		panic(err)
+	}
+	return EncryptingPrivateKey{k}
+}
+
+// PublicKey returns the public key which corresponds with this private key.
+func (k EncryptingPrivateKey) PublicKey() EncryptingPublicKey {
+	return EncryptingPublicKey{k.inner.PublicKey()}
+}
+
+// MarshalText implements the encoding.TextMarshaler interface.
+func (k EncryptingPrivateKey) MarshalText() ([]byte, error) {
+	return encodeWithPrefix(encPrivKeyPrefix, k.inner.Bytes()), nil
+}
+
+// Bytes returns the raw bytes of the EncryptingPrivateKey.
+func (k EncryptingPrivateKey) Bytes() []byte {
+	return k.inner.Bytes()
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface.
+func (k *EncryptingPrivateKey) UnmarshalText(b []byte) error {
+	b, err := decodeWithPrefix(encPrivKeyPrefix, b)
+	if err != nil {
+		return fmt.Errorf("unmarshaling: %w", err)
+	}
+
+	if k.inner, err = x25519.NewPrivateKey(b); err != nil {
+		return fmt.Errorf("converting bytes to private key: %w", err)
+	}
+
+	return nil
+}
+
+func (k EncryptingPrivateKey) String() string {
+	b, err := k.MarshalText()
+	if err != nil {
+		panic(err)
+	}
+	return string(b)
+}

go/nebula/keys.go

@@ -1,72 +0,0 @@
-package nebula
-
-import (
-	"crypto/ed25519"
-	"crypto/rand"
-	"encoding/json"
-	"fmt"
-
-	"github.com/slackhq/nebula/cert"
-)
-
-// SigningPrivateKey wraps an ed25519.PrivateKey to provide convenient JSON
-// (un)marshaling methods.
-type SigningPrivateKey ed25519.PrivateKey
-
-// MarshalJSON implements the json.Marshaler interface.
-func (k SigningPrivateKey) MarshalJSON() ([]byte, error) {
-	pemStr := cert.MarshalEd25519PrivateKey(ed25519.PrivateKey(k))
-	return json.Marshal(string(pemStr))
-}
-
-// UnmarshalJSON implements the json.Unmarshaler interface.
-func (k *SigningPrivateKey) UnmarshalJSON(b []byte) error {
-	var pemStr string
-	if err := json.Unmarshal(b, &pemStr); err != nil {
-		return fmt.Errorf("unmarshaling into string: %w", err)
-	}
-
-	key, _, err := cert.UnmarshalEd25519PrivateKey([]byte(pemStr))
-	if err != nil {
-		return fmt.Errorf("unmarshaling from PEM: %w", err)
-	}
-
-	*k = SigningPrivateKey(key)
-	return nil
-}
-
-// SigningPublicKey wraps an ed25519.PublicKey to provide convenient JSON
-// (un)marshaling methods.
-type SigningPublicKey ed25519.PublicKey
-
-// MarshalJSON implements the json.Marshaler interface.
-func (k SigningPublicKey) MarshalJSON() ([]byte, error) {
-	pemStr := cert.MarshalEd25519PublicKey(ed25519.PublicKey(k))
-	return json.Marshal(string(pemStr))
-}
-
-// MarshalJSON implements the json.Unmarshaler interface.
-func (k *SigningPublicKey) UnmarshalJSON(b []byte) error {
-	var pemStr string
-	if err := json.Unmarshal(b, &pemStr); err != nil {
-		return fmt.Errorf("unmarshaling into string: %w", err)
-	}
-
-	key, _, err := cert.UnmarshalEd25519PublicKey([]byte(pemStr))
-	if err != nil {
-		return fmt.Errorf("unmarshaling from PEM: %w", err)
-	}
-
-	*k = SigningPublicKey(key)
-	return nil
-}
-
-// GenerateSigningPair generates and returns a new key pair which can be used
-// for signing arbitrary blobs of bytes.
-func GenerateSigningPair() (SigningPublicKey, SigningPrivateKey) {
-	pub, priv, err := ed25519.GenerateKey(rand.Reader)
-	if err != nil {
-		panic(fmt.Errorf("generating ed25519 key: %w", err))
-	}
-	return SigningPublicKey(pub), SigningPrivateKey(priv)
-}

go/nebula/nebula.go

@@ -3,35 +3,32 @@
 package nebula
 
 import (
-	"crypto/rand"
 	"fmt"
-	"io"
 	"net"
 	"time"
 
 	"github.com/slackhq/nebula/cert"
-	"golang.org/x/crypto/curve25519"
 )
 
 // HostPublicCredentials contains certificate and signing public keys which are
 // able to be broadcast publicly.
 type HostPublicCredentials struct {
-	CertPEM    string
+	Cert       cert.NebulaCertificate
 	SigningKey SigningPublicKey
 }
 
 // HostPrivateCredentials contains the private key files which will
 // need to be present on a particular host.
 type HostPrivateCredentials struct {
-	PrivateKeyPEM     string
-	SigningPrivateKey SigningPrivateKey
+	EncryptingPrivateKey EncryptingPrivateKey
+	SigningPrivateKey    SigningPrivateKey
 }
 
 // CAPublicCredentials contains certificate and signing public keys which are
 // able to be broadcast publicly. The signing public key is the same one which
 // is embedded into the certificate.
 type CAPublicCredentials struct {
-	CertPEM    string
+	Cert       cert.NebulaCertificate
 	SigningKey SigningPublicKey
 }
 
@@ -42,33 +39,28 @@ type CACredentials struct {
 	SigningPrivateKey SigningPrivateKey
 }
 
-// NewHostCertPEM generates and signs a new host certificate containing the
-// given public key.
-func NewHostCertPEM(
-	caCreds CACredentials, hostPubPEM string, hostName string, ip net.IP,
+// NewHostCert generates and signs a new host certificate containing the given
+// public key.
+func NewHostCert(
+	caCreds CACredentials,
+	hostPub EncryptingPublicKey,
+	hostName string,
+	ip net.IP,
 ) (
-	string, error,
+	cert.NebulaCertificate, error,
 ) {
-	hostPub, _, err := cert.UnmarshalX25519PublicKey([]byte(hostPubPEM))
-	if err != nil {
-		return "", fmt.Errorf("unmarshaling public key PEM: %w", err)
-	}
-
-	caCert, _, err := cert.UnmarshalNebulaCertificateFromPEM([]byte(caCreds.Public.CertPEM))
-	if err != nil {
-		return "", fmt.Errorf("unmarshaling ca.crt: %w", err)
-	}
+	caCert := caCreds.Public.Cert
 
 	issuer, err := caCert.Sha256Sum()
 	if err != nil {
-		return "", fmt.Errorf("getting ca.crt issuer: %w", err)
+		return cert.NebulaCertificate{}, fmt.Errorf("getting ca.crt issuer: %w", err)
 	}
 
 	expireAt := caCert.Details.NotAfter.Add(-1 * time.Second)
 
 	subnet := caCert.Details.Subnets[0]
 	if !subnet.Contains(ip) {
-		return "", fmt.Errorf("invalid ip %q, not contained by network subnet %q", ip, subnet)
+		return cert.NebulaCertificate{}, fmt.Errorf("invalid ip %q, not contained by network subnet %q", ip, subnet)
 	}
 
 	hostCert := cert.NebulaCertificate{
@@ -80,26 +72,21 @@ func NewHostCertPEM(
 			}},
 			NotBefore: time.Now(),
 			NotAfter:  expireAt,
-			PublicKey: hostPub,
+			PublicKey: hostPub.Bytes(),
 			IsCA:      false,
 			Issuer:    issuer,
 		},
 	}
 
-	if err := hostCert.CheckRootConstrains(caCert); err != nil {
-		return "", fmt.Errorf("validating certificate constraints: %w", err)
+	if err := hostCert.CheckRootConstrains(&caCert); err != nil {
+		return cert.NebulaCertificate{}, fmt.Errorf("validating certificate constraints: %w", err)
 	}
 
 	if err := signCert(&hostCert, caCreds.SigningPrivateKey); err != nil {
-		return "", fmt.Errorf("signing host cert with ca.key: %w", err)
+		return cert.NebulaCertificate{}, fmt.Errorf("signing host cert with ca.key: %w", err)
 	}
 
-	hostCertPEM, err := hostCert.MarshalToPEM()
-	if err != nil {
-		return "", fmt.Errorf("marshalling host.crt: %w", err)
-	}
-
-	return string(hostCertPEM), nil
+	return hostCert, nil
 }
 
 // NewHostCredentials generates a new key/cert for a nebula host using the CA
@@ -110,39 +97,27 @@ func NewHostCredentials(
 	pub HostPublicCredentials, priv HostPrivateCredentials, err error,
 ) {
 
-	// The logic here is largely based on
-	// https://github.com/slackhq/nebula/blob/v1.4.0/cmd/nebula-cert/sign.go
+	var (
+		encPrivKey = NewEncryptingPrivateKey()
+		encPubKey  = encPrivKey.PublicKey()
 
-	var hostPub, hostKey []byte
-	{
-		var pubkey, privkey [32]byte
-		if _, err = io.ReadFull(rand.Reader, privkey[:]); err != nil {
-			err = fmt.Errorf("reading random bytes to form private key: %w", err)
-			return
-		}
-		curve25519.ScalarBaseMult(&pubkey, &privkey)
-		hostPub, hostKey = pubkey[:], privkey[:]
-	}
+		signingPubKey, signingPrivKey = GenerateSigningPair()
+	)
 
-	signingPubKey, signingPrivKey := GenerateSigningPair()
-
-	hostPubPEM := cert.MarshalX25519PublicKey(hostPub)
-	hostKeyPEM := cert.MarshalX25519PrivateKey(hostKey)
-
-	hostCertPEM, err := NewHostCertPEM(caCreds, string(hostPubPEM), hostName, ip)
+	hostCert, err := NewHostCert(caCreds, encPubKey, hostName, ip)
 	if err != nil {
 		err = fmt.Errorf("creating host certificate: %w", err)
 		return
 	}
 
 	pub = HostPublicCredentials{
-		CertPEM:    hostCertPEM,
+		Cert:       hostCert,
 		SigningKey: signingPubKey,
 	}
 
 	priv = HostPrivateCredentials{
-		PrivateKeyPEM:     string(hostKeyPEM),
-		SigningPrivateKey: signingPrivKey,
+		EncryptingPrivateKey: encPrivKey,
+		SigningPrivateKey:    signingPrivKey,
 	}
 
 	return
@@ -151,14 +126,11 @@ func NewHostCredentials(
 // NewCACredentials generates a CACredentials. The domain should be the network's root domain,
 // and is included in the signing certificate's Name field.
 func NewCACredentials(domain string, subnet *net.IPNet) (CACredentials, error) {
-
-	// The logic here is largely based on
-	// https://github.com/slackhq/nebula/blob/v1.4.0/cmd/nebula-cert/ca.go
-
-	signingPubKey, signingPrivKey := GenerateSigningPair()
-
-	now := time.Now()
-	expireAt := now.Add(2 * 365 * 24 * time.Hour)
+	var (
+		signingPubKey, signingPrivKey = GenerateSigningPair()
+		now                           = time.Now()
+		expireAt                      = now.Add(2 * 365 * 24 * time.Hour)
+	)
 
 	caCert := cert.NebulaCertificate{
 		Details: cert.NebulaCertificateDetails{
@@ -175,33 +147,11 @@ func NewCACredentials(domain string, subnet *net.IPNet) (CACredentials, error) {
 		return CACredentials{}, fmt.Errorf("signing caCert: %w", err)
 	}
 
-	certPEM, err := caCert.MarshalToPEM()
-	if err != nil {
-		return CACredentials{}, fmt.Errorf("marshaling caCert: %w", err)
-	}
-
 	return CACredentials{
 		Public: CAPublicCredentials{
-			CertPEM:    string(certPEM),
+			Cert:       caCert,
 			SigningKey: signingPubKey,
 		},
 		SigningPrivateKey: signingPrivKey,
 	}, nil
 }
-
-// IPFromHostCertPEM is a convenience function for parsing the IP of a host out
-// of its nebula cert.
-func IPFromHostCertPEM(hostCertPEM string) (net.IP, error) {
-
-	hostCert, _, err := cert.UnmarshalNebulaCertificateFromPEM([]byte(hostCertPEM))
-	if err != nil {
-		return nil, fmt.Errorf("unmarshaling host certificate as PEM: %w", err)
-	}
-
-	ips := hostCert.Details.Ips
-	if len(ips) == 0 {
-		return nil, fmt.Errorf("malformed nebula host cert: no IPs")
-	}
-
-	return ips[0].IP, nil
-}

go/nebula/signing_key.go

@@ -0,0 +1,78 @@
+package nebula
+
+import (
+	"crypto/ed25519"
+	"crypto/rand"
+	"fmt"
+)
+
+var (
+	sigPrivKeyPrefix = []byte("s0")
+	sigPubKeyPrefix  = []byte("S0")
+)
+
+// SigningPrivateKey wraps an ed25519.PrivateKey to provide convenient text
+// (un)marshaling methods.
+type SigningPrivateKey ed25519.PrivateKey
+
+// MarshalText implements the encoding.TextMarshaler interface.
+func (k SigningPrivateKey) MarshalText() ([]byte, error) {
+	return encodeWithPrefix(sigPrivKeyPrefix, k), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface.
+func (k *SigningPrivateKey) UnmarshalText(b []byte) error {
+	b, err := decodeWithPrefix(sigPrivKeyPrefix, b)
+	if err != nil {
+		return fmt.Errorf("unmarshaling: %w", err)
+	}
+
+	*k = SigningPrivateKey(b)
+	return nil
+}
+
+func (k SigningPrivateKey) String() string {
+	b, err := k.MarshalText()
+	if err != nil {
+		panic(err)
+	}
+	return string(b)
+}
+
+// SigningPublicKey wraps an ed25519.PublicKey to provide convenient text
+// (un)marshaling methods.
+type SigningPublicKey ed25519.PublicKey
+
+// MarshalText implements the encoding.TextMarshaler interface.
+func (pk SigningPublicKey) MarshalText() ([]byte, error) {
+	return encodeWithPrefix(sigPubKeyPrefix, pk), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface.
+func (pk *SigningPublicKey) UnmarshalText(b []byte) error {
+	b, err := decodeWithPrefix(sigPubKeyPrefix, b)
+	if err != nil {
+		return fmt.Errorf("unmarshaling: %w", err)
+	}
+
+	*pk = SigningPublicKey(b)
+	return nil
+}
+
+func (pk SigningPublicKey) String() string {
+	b, err := pk.MarshalText()
+	if err != nil {
+		panic(err)
+	}
+	return string(b)
+}
+
+// GenerateSigningPair generates and returns a new key pair which can be used
+// for signing arbitrary blobs of bytes.
+func GenerateSigningPair() (SigningPublicKey, SigningPrivateKey) {
+	pub, priv, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		panic(fmt.Errorf("generating ed25519 key: %w", err))
+	}
+	return SigningPublicKey(pub), SigningPrivateKey(priv)
+}

go/nebula/util.go

@@ -0,0 +1,25 @@
+package nebula
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+
+	"github.com/jxskiss/base62"
+)
+
+func encodeWithPrefix(prefix []byte, b []byte) []byte {
+	res := make([]byte, 0, len(prefix)+len(b)*4)
+	res = append(res, prefix...)
+	res = base62.EncodeToBuf(res, b)
+	return res
+}
+
+func decodeWithPrefix(prefix []byte, b []byte) ([]byte, error) {
+	if len(b) < len(prefix) {
+		return nil, errors.New("input is too short")
+	} else if !bytes.HasPrefix(b, prefix) {
+		return nil, fmt.Errorf("missing expected prefix %q", prefix)
+	}
+	return base62.Decode(b[len(prefix):])
+}

nix/garage.nix

@@ -1,20 +1,15 @@
 rec {
 
-  version = "0.8.1";
+  version = "1.0.0";
 
   src = builtins.fetchGit {
     name = "garage-v${version}";
     url = "https://git.deuxfleurs.fr/Deuxfleurs/garage.git";
-    rev = "76230f20282e73a5a5afa33af68152acaf732cf5";
+    rev = "ff093ddbb8485409f389abe7b5e569cb38d222d2";
   };
 
-  # TODO compiling garage broke using 24.05, so for now use the pkgs pinned in
-  # the garage repo. Probably will revisit this when garage gets upgraded
-  # anyway.
-  pkgsSrc = (import "${src}/nix/common.nix").pkgsSrc;
-
   package = {
-    #pkgsSrc,
+    pkgsSrc,
     buildSystem,
     hostSystem,
   }: let
@@ -29,6 +24,16 @@ rec {
 
       release = true;
       git_version = version;
+
+      # subset of the default release features, as defined in:
+      # https://git.deuxfleurs.fr/Deuxfleurs/garage/src/commit/ff093ddbb8485409f389abe7b5e569cb38d222d2/nix/compile.nix#L171
+      features = [
+        "garage/bundled-libs"
+        "garage/lmdb"
+        "garage/sqlite"
+        "garage/metrics"
+        "garage/syslog"
+      ];
     };
 
   in

nix/pkgs.nix

@@ -38,7 +38,9 @@ rec {
   supportedSystems = [
     "x86_64-linux"
     "aarch64-linux"
-    #"armv7l-linux-musl" # rpi, I think?
+
+    # rpi, TODO remember why this is disabled, try to re-enable it
+    #"armv7l-linux-musl"
     "i686-linux"
   ];
 

tests/entrypoint.sh

@@ -68,14 +68,15 @@ for file in $test_files; do
     if [ -z "$VERBOSE" ]; then
         output="$TMPDIR/$file.log"
         mkdir -p "$(dirname "$output")"
+        exec 3>"$output"
     else
-        output=/dev/stdout
+        exec 3>&1
     fi
 
     (
         export TEST_CASE_FILE="$file"
 
-        if ! $SHELL -e -x "$root/cases/$file" >"$output" 2>&1; then
+        if ! $SHELL -e -x "$root/cases/$file" >&3 2>&1; then
             echo "$file FAILED"
             if [ -z "$VERBOSE" ]; then
                 echo "output of test is as follows"

tests/utils/with-1-data-1-empty-node-cluster.sh

@@ -40,13 +40,13 @@ if [ ! -d "$XDG_RUNTIME_DIR/isle" ]; then
       allocations:
         - data_path: a/data
           meta_path: a/meta
-          capacity: 100
+          capacity: 1
         - data_path: b/data
           meta_path: b/meta
-          capacity: 100
+          capacity: 1
         - data_path: c/data
           meta_path: c/meta
-          capacity: 100
+          capacity: 1
 EOF
 
     echo "Creating 1-data-1-empty network"