Upgrade garage to v1.0.0

This required switching all garage admin API calls to the new v1
versions, and redoing how the global bucket key is created so it is
created via the "create key" API call.

AppDir/etc/daemon.yml

@@ -69,7 +69,7 @@ storage:
   # the meta directories can be anywhere (ideally on an SSD).
   #
   # Capacity declares how many gigabytes can be stored in each allocation, and
-  # is required. It must be a multiple of 100.
+  # is required.
   #
   # The ports are all _optional_, and will be automatically assigned if they are
   # not specified. If ports any ports are specified then all should be

default.nix

@@ -90,7 +90,7 @@ in rec {
 
     inherit buildSystem;
     hostSystem = "${hostPlatform.cpu.name}-unknown-${hostPlatform.kernel.name}-musl";
-    #pkgsSrc = pkgsNix.src;
+    pkgsSrc = pkgsNix.src;
 
   };
 

docs/operator/contributing-storage.md

@@ -35,7 +35,7 @@ storage:
       meta_path: /mnt/drive1/isle/meta
       capacity: 1200
 
-    # 100 GB (the minimum) are being shared from drive2
+    # 100 GB are being shared from drive2
     - data_path: /mnt/drive2/isle/data
       meta_path: /mnt/drive2/isle/meta
       capacity: 100

go/bootstrap/bootstrap.go

@@ -30,10 +30,13 @@ func AppDirPath(appDirPath string) string {
 
 // Garage contains parameters needed to connect to and use the garage cluster.
 type Garage struct {
-	// TODO RPCSecret and GlobalBucketS3APICredentials are duplicated here and
+	// TODO this should be part of some new configuration section related to
-	// in AdminCreationParams, might as well just use them from there
+	// secrets which may or may not be granted to this host
-	RPCSecret                    string
+	RPCSecret string
-	AdminToken                   string
+
+	AdminToken string
+
+	// TODO this should be part of admin.CreationParams
 	GlobalBucketS3APICredentials garage.S3APICredentials
 }
 

go/cmd/entrypoint/admin.go

@@ -149,9 +149,8 @@ var subCmdAdminCreateNetwork = subCmd{
 		}
 
 		garageBootstrap := bootstrap.Garage{
-			RPCSecret:                    randStr(32),
+			RPCSecret:  randStr(32),
-			AdminToken:                   randStr(32),
+			AdminToken: randStr(32),
-			GlobalBucketS3APICredentials: garage.NewS3APICredentials(),
 		}
 
 		hostBootstrap, err := bootstrap.New(
@@ -216,7 +215,17 @@ var subCmdAdminCreateNetwork = subCmd{
 		}
 
 		logger.Info(ctx, "initializing garage shared global bucket")
-		err = garageInitializeGlobalBucket(ctx, logger, hostBootstrap, daemonConfig)
+		garageGlobalBucketCreds, err := garageInitializeGlobalBucket(
+			ctx, logger, hostBootstrap, daemonConfig,
+		)
+
+		hostBootstrap.Garage.GlobalBucketS3APICredentials = garageGlobalBucketCreds
+
+		// rewrite the bootstrap now that the global bucket creds have been
+		// added to it.
+		if err := writeBootstrapToDataDir(hostBootstrap); err != nil {
+			return fmt.Errorf("writing bootstrap file: %w", err)
+		}
 
 		if cErr := (garage.AdminClientError{}); errors.As(err, &cErr) && cErr.StatusCode == 409 {
 			return fmt.Errorf("shared global bucket has already been created, are the storage allocations from a previously initialized isle being used?")

go/cmd/entrypoint/garage_util.go

@@ -2,10 +2,10 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"isle/bootstrap"
 	"isle/daemon"
 	"isle/garage"
-	"fmt"
 	"net"
 	"path/filepath"
 	"strconv"
@@ -184,51 +184,50 @@ func garageInitializeGlobalBucket(
 	logger *mlog.Logger,
 	hostBootstrap bootstrap.Bootstrap,
 	daemonConfig daemon.Config,
-) error {
+) (
+	garage.S3APICredentials, error,
+) {
+	adminClient := newGarageAdminClient(logger, hostBootstrap, daemonConfig)
 
-	var (
+	var createKeyRes struct {
-		adminClient       = newGarageAdminClient(logger, hostBootstrap, daemonConfig)
+		ID     string `json:"accessKeyId"`
-		globalBucketCreds = hostBootstrap.Garage.GlobalBucketS3APICredentials
+		Secret string `json:"secretAccessKey"`
-	)
+	}
 
 	// first attempt to import the key
-	err := adminClient.Do(ctx, nil, "POST", "/v0/key/import", map[string]string{
+	err := adminClient.Do(
-		"accessKeyId":     globalBucketCreds.ID,
+		ctx, &createKeyRes, "POST", "/v1/key", map[string]string{
-		"secretAccessKey": globalBucketCreds.Secret,
+			"name": "shared-global-bucket-key",
-		"name":            "shared-global-bucket-key",
+		},
-	})
+	)
 
 	if err != nil {
-		return fmt.Errorf("importing global bucket key into garage: %w", err)
+		return garage.S3APICredentials{}, fmt.Errorf(
+			"importing global bucket key into garage: %w", err,
+		)
 	}
 
 	// create global bucket
-	err = adminClient.Do(ctx, nil, "POST", "/v0/bucket", map[string]string{
+	var createBucketRes struct {
-		"globalAlias": garage.GlobalBucket,
-	})
-
-	if err != nil {
-		return fmt.Errorf("creating global bucket: %w", err)
-	}
-
-	// retrieve newly created bucket's id
-	var getBucketRes struct {
 		ID string `json:"id"`
 	}
 
 	err = adminClient.Do(
-		ctx, &getBucketRes,
+		ctx, &createBucketRes, "POST", "/v1/bucket", map[string]string{
-		"GET", "/v0/bucket?globalAlias="+garage.GlobalBucket, nil,
+			"globalAlias": garage.GlobalBucket,
+		},
 	)
 
 	if err != nil {
-		return fmt.Errorf("fetching global bucket id: %w", err)
+		return garage.S3APICredentials{}, fmt.Errorf(
+			"creating global bucket: %w", err,
+		)
 	}
 
 	// allow shared global bucket key to perform all operations
-	err = adminClient.Do(ctx, nil, "POST", "/v0/bucket/allow", map[string]interface{}{
+	err = adminClient.Do(ctx, nil, "POST", "/v1/bucket/allow", map[string]interface{}{
-		"bucketId":    getBucketRes.ID,
+		"bucketId":    createBucketRes.ID,
-		"accessKeyId": globalBucketCreds.ID,
+		"accessKeyId": createKeyRes.ID,
 		"permissions": map[string]bool{
 			"read":  true,
 			"write": true,
@@ -236,10 +235,15 @@ func garageInitializeGlobalBucket(
 	})
 
 	if err != nil {
-		return fmt.Errorf("granting permissions to shared global bucket key: %w", err)
+		return garage.S3APICredentials{}, fmt.Errorf(
+			"granting permissions to shared global bucket key: %w", err,
+		)
 	}
 
-	return nil
+	return garage.S3APICredentials{
+		ID:     createKeyRes.ID,
+		Secret: createKeyRes.Secret,
+	}, nil
 }
 
 func garageApplyLayout(
@@ -257,15 +261,16 @@ func garageApplyLayout(
 	)
 
 	type peerLayout struct {
+		ID       string   `json:"id"`
 		Capacity int      `json:"capacity"`
 		Zone     string   `json:"zone"`
 		Tags     []string `json:"tags"`
 	}
 
 	{
-		clusterLayout := map[string]peerLayout{}
+		clusterLayout := make([]peerLayout, len(allocs))
 
-		for _, alloc := range allocs {
+		for i, alloc := range allocs {
 
 			id := bootstrapGarageHostForAlloc(thisHost, alloc).ID
 
@@ -274,25 +279,26 @@ func garageApplyLayout(
 				zone = alloc.Zone
 			}
 
-			clusterLayout[id] = peerLayout{
+			clusterLayout[i] = peerLayout{
-				Capacity: alloc.Capacity,
+				ID:       id,
+				Capacity: alloc.Capacity * 1_000_000_000,
 				Zone:     zone,
 				Tags:     []string{},
 			}
 		}
 
-		err := adminClient.Do(ctx, nil, "POST", "/v0/layout", clusterLayout)
+		err := adminClient.Do(ctx, nil, "POST", "/v1/layout", clusterLayout)
 		if err != nil {
 			return fmt.Errorf("staging layout changes: %w", err)
 		}
 	}
 
 	var clusterLayout struct {
-		Version           int                   `json:"version"`
+		Version           int          `json:"version"`
-		StagedRoleChanges map[string]peerLayout `json:"stagedRoleChanges"`
+		StagedRoleChanges []peerLayout `json:"stagedRoleChanges"`
 	}
 
-	if err := adminClient.Do(ctx, &clusterLayout, "GET", "/v0/layout", nil); err != nil {
+	if err := adminClient.Do(ctx, &clusterLayout, "GET", "/v1/layout", nil); err != nil {
 		return fmt.Errorf("retrieving staged layout change: %w", err)
 	}
 
@@ -306,7 +312,7 @@ func garageApplyLayout(
 		Version: clusterLayout.Version + 1,
 	}
 
-	err := adminClient.Do(ctx, nil, "POST", "/v0/layout/apply", applyClusterLayout)
+	err := adminClient.Do(ctx, nil, "POST", "/v1/layout/apply", applyClusterLayout)
 	if err != nil {
 		return fmt.Errorf("applying new layout (new version:%d): %w", applyClusterLayout.Version, err)
 	}

go/garage/admin_client.go

@@ -139,12 +139,12 @@ func (c *AdminClient) Wait(ctx context.Context) error {
 		}
 
 		var clusterStatus struct {
-			KnownNodes map[string]struct {
+			Nodes []struct {
-				IsUp bool `json:"is_up"`
+				IsUp bool `json:"isUp"`
-			} `json:"knownNodes"`
+			} `json:"nodes"`
 		}
 
-		err := c.Do(ctx, &clusterStatus, "GET", "/v0/status", nil)
+		err := c.Do(ctx, &clusterStatus, "GET", "/v1/status", nil)
 
 		if ctxErr := ctx.Err(); ctxErr != nil {
 			return ctxErr
@@ -156,14 +156,14 @@ func (c *AdminClient) Wait(ctx context.Context) error {
 
 		var numUp int
 
-		for _, knownNode := range clusterStatus.KnownNodes {
+		for _, node := range clusterStatus.Nodes {
-			if knownNode.IsUp {
+			if node.IsUp {
 				numUp++
 			}
 		}
 
 		ctx := mctx.Annotate(ctx,
-			"numKnownNodes", len(clusterStatus.KnownNodes),
+			"numNodes", len(clusterStatus.Nodes),
 			"numUp", numUp,
 		)
 

go/garage/client.go

@@ -35,14 +35,6 @@ type S3APICredentials struct {
 	Secret string
 }
 
-// NewS3APICredentials returns a new usable instance of S3APICredentials.
-func NewS3APICredentials() S3APICredentials {
-	return S3APICredentials{
-		ID:     randStr(8),
-		Secret: randStr(32),
-	}
-}
-
 // NewS3APIClient returns a minio client configured to use the given garage S3 API
 // endpoint.
 func NewS3APIClient(addr string, creds S3APICredentials) S3APIClient {

nix/garage.nix

@@ -1,20 +1,15 @@
 rec {
 
-  version = "0.8.1";
+  version = "1.0.0";
 
   src = builtins.fetchGit {
     name = "garage-v${version}";
     url = "https://git.deuxfleurs.fr/Deuxfleurs/garage.git";
-    rev = "76230f20282e73a5a5afa33af68152acaf732cf5";
+    rev = "ff093ddbb8485409f389abe7b5e569cb38d222d2";
   };
 
-  # TODO compiling garage broke using 24.05, so for now use the pkgs pinned in
-  # the garage repo. Probably will revisit this when garage gets upgraded
-  # anyway.
-  pkgsSrc = (import "${src}/nix/common.nix").pkgsSrc;
-
   package = {
-    #pkgsSrc,
+    pkgsSrc,
     buildSystem,
     hostSystem,
   }: let
@@ -29,6 +24,16 @@ rec {
 
       release = true;
       git_version = version;
+
+      # subset of the default release features, as defined in:
+      # https://git.deuxfleurs.fr/Deuxfleurs/garage/src/commit/ff093ddbb8485409f389abe7b5e569cb38d222d2/nix/compile.nix#L171
+      features = [
+        "garage/bundled-libs"
+        "garage/lmdb"
+        "garage/sqlite"
+        "garage/metrics"
+        "garage/syslog"
+      ];
     };
 
   in

nix/pkgs.nix

@@ -38,7 +38,9 @@ rec {
   supportedSystems = [
     "x86_64-linux"
     "aarch64-linux"
-    #"armv7l-linux-musl" # rpi, I think?
+
+    # rpi, TODO remember why this is disabled, try to re-enable it
+    #"armv7l-linux-musl"
     "i686-linux"
   ];
 

tests/utils/with-1-data-1-empty-node-cluster.sh

@@ -40,13 +40,13 @@ if [ ! -d "$XDG_RUNTIME_DIR/isle" ]; then
       allocations:
         - data_path: a/data
           meta_path: a/meta
-          capacity: 100
+          capacity: 1
         - data_path: b/data
           meta_path: b/meta
-          capacity: 100
+          capacity: 1
         - data_path: c/data
           meta_path: c/meta
-          capacity: 100
+          capacity: 1
 EOF
 
     echo "Creating 1-data-1-empty network"