From 68f417b5bace3047eb5d582761e5ac00e9c2a788 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Tue, 11 Jun 2024 14:54:26 +0200 Subject: [PATCH] Upgrade garage to v1.0.0 This required switching all garage admin API calls to the new v1 versions, and redoing how the global bucket key is created so it is created via the "create key" API call. --- AppDir/etc/daemon.yml | 2 +- default.nix | 2 +- docs/operator/contributing-storage.md | 2 +- go/bootstrap/bootstrap.go | 11 ++- go/cmd/entrypoint/admin.go | 17 +++- go/cmd/entrypoint/garage_util.go | 84 ++++++++++--------- go/garage/admin_client.go | 14 ++-- go/garage/client.go | 8 -- nix/garage.nix | 21 +++-- nix/pkgs.nix | 4 +- .../utils/with-1-data-1-empty-node-cluster.sh | 6 +- 11 files changed, 94 insertions(+), 77 deletions(-) diff --git a/AppDir/etc/daemon.yml b/AppDir/etc/daemon.yml index 1baf983..2ff913d 100644 --- a/AppDir/etc/daemon.yml +++ b/AppDir/etc/daemon.yml @@ -69,7 +69,7 @@ storage: # the meta directories can be anywhere (ideally on an SSD). # # Capacity declares how many gigabytes can be stored in each allocation, and - # is required. It must be a multiple of 100. + # is required. # # The ports are all _optional_, and will be automatically assigned if they are # not specified. If ports any ports are specified then all should be diff --git a/default.nix b/default.nix index 20d2007..f752824 100644 --- a/default.nix +++ b/default.nix @@ -90,7 +90,7 @@ in rec { inherit buildSystem; hostSystem = "${hostPlatform.cpu.name}-unknown-${hostPlatform.kernel.name}-musl"; - #pkgsSrc = pkgsNix.src; + pkgsSrc = pkgsNix.src; }; diff --git a/docs/operator/contributing-storage.md b/docs/operator/contributing-storage.md index 7633a36..c5e2c0b 100644 --- a/docs/operator/contributing-storage.md +++ b/docs/operator/contributing-storage.md @@ -35,7 +35,7 @@ storage: meta_path: /mnt/drive1/isle/meta capacity: 1200 - # 100 GB (the minimum) are being shared from drive2 + # 100 GB are being shared from drive2 - data_path: /mnt/drive2/isle/data meta_path: /mnt/drive2/isle/meta capacity: 100 diff --git a/go/bootstrap/bootstrap.go b/go/bootstrap/bootstrap.go index 6ebc786..6470c6c 100644 --- a/go/bootstrap/bootstrap.go +++ b/go/bootstrap/bootstrap.go @@ -30,10 +30,13 @@ func AppDirPath(appDirPath string) string { // Garage contains parameters needed to connect to and use the garage cluster. type Garage struct { - // TODO RPCSecret and GlobalBucketS3APICredentials are duplicated here and - // in AdminCreationParams, might as well just use them from there - RPCSecret string - AdminToken string + // TODO this should be part of some new configuration section related to + // secrets which may or may not be granted to this host + RPCSecret string + + AdminToken string + + // TODO this should be part of admin.CreationParams GlobalBucketS3APICredentials garage.S3APICredentials } diff --git a/go/cmd/entrypoint/admin.go b/go/cmd/entrypoint/admin.go index 8e31753..ffeb7e7 100644 --- a/go/cmd/entrypoint/admin.go +++ b/go/cmd/entrypoint/admin.go @@ -149,9 +149,8 @@ var subCmdAdminCreateNetwork = subCmd{ } garageBootstrap := bootstrap.Garage{ - RPCSecret: randStr(32), - AdminToken: randStr(32), - GlobalBucketS3APICredentials: garage.NewS3APICredentials(), + RPCSecret: randStr(32), + AdminToken: randStr(32), } hostBootstrap, err := bootstrap.New( @@ -216,7 +215,17 @@ var subCmdAdminCreateNetwork = subCmd{ } logger.Info(ctx, "initializing garage shared global bucket") - err = garageInitializeGlobalBucket(ctx, logger, hostBootstrap, daemonConfig) + garageGlobalBucketCreds, err := garageInitializeGlobalBucket( + ctx, logger, hostBootstrap, daemonConfig, + ) + + hostBootstrap.Garage.GlobalBucketS3APICredentials = garageGlobalBucketCreds + + // rewrite the bootstrap now that the global bucket creds have been + // added to it. + if err := writeBootstrapToDataDir(hostBootstrap); err != nil { + return fmt.Errorf("writing bootstrap file: %w", err) + } if cErr := (garage.AdminClientError{}); errors.As(err, &cErr) && cErr.StatusCode == 409 { return fmt.Errorf("shared global bucket has already been created, are the storage allocations from a previously initialized isle being used?") diff --git a/go/cmd/entrypoint/garage_util.go b/go/cmd/entrypoint/garage_util.go index 5be8d46..dff0b02 100644 --- a/go/cmd/entrypoint/garage_util.go +++ b/go/cmd/entrypoint/garage_util.go @@ -2,10 +2,10 @@ package main import ( "context" + "fmt" "isle/bootstrap" "isle/daemon" "isle/garage" - "fmt" "net" "path/filepath" "strconv" @@ -184,51 +184,50 @@ func garageInitializeGlobalBucket( logger *mlog.Logger, hostBootstrap bootstrap.Bootstrap, daemonConfig daemon.Config, -) error { +) ( + garage.S3APICredentials, error, +) { + adminClient := newGarageAdminClient(logger, hostBootstrap, daemonConfig) - var ( - adminClient = newGarageAdminClient(logger, hostBootstrap, daemonConfig) - globalBucketCreds = hostBootstrap.Garage.GlobalBucketS3APICredentials - ) + var createKeyRes struct { + ID string `json:"accessKeyId"` + Secret string `json:"secretAccessKey"` + } // first attempt to import the key - err := adminClient.Do(ctx, nil, "POST", "/v0/key/import", map[string]string{ - "accessKeyId": globalBucketCreds.ID, - "secretAccessKey": globalBucketCreds.Secret, - "name": "shared-global-bucket-key", - }) + err := adminClient.Do( + ctx, &createKeyRes, "POST", "/v1/key", map[string]string{ + "name": "shared-global-bucket-key", + }, + ) if err != nil { - return fmt.Errorf("importing global bucket key into garage: %w", err) + return garage.S3APICredentials{}, fmt.Errorf( + "importing global bucket key into garage: %w", err, + ) } // create global bucket - err = adminClient.Do(ctx, nil, "POST", "/v0/bucket", map[string]string{ - "globalAlias": garage.GlobalBucket, - }) - - if err != nil { - return fmt.Errorf("creating global bucket: %w", err) - } - - // retrieve newly created bucket's id - var getBucketRes struct { + var createBucketRes struct { ID string `json:"id"` } err = adminClient.Do( - ctx, &getBucketRes, - "GET", "/v0/bucket?globalAlias="+garage.GlobalBucket, nil, + ctx, &createBucketRes, "POST", "/v1/bucket", map[string]string{ + "globalAlias": garage.GlobalBucket, + }, ) if err != nil { - return fmt.Errorf("fetching global bucket id: %w", err) + return garage.S3APICredentials{}, fmt.Errorf( + "creating global bucket: %w", err, + ) } // allow shared global bucket key to perform all operations - err = adminClient.Do(ctx, nil, "POST", "/v0/bucket/allow", map[string]interface{}{ - "bucketId": getBucketRes.ID, - "accessKeyId": globalBucketCreds.ID, + err = adminClient.Do(ctx, nil, "POST", "/v1/bucket/allow", map[string]interface{}{ + "bucketId": createBucketRes.ID, + "accessKeyId": createKeyRes.ID, "permissions": map[string]bool{ "read": true, "write": true, @@ -236,10 +235,15 @@ func garageInitializeGlobalBucket( }) if err != nil { - return fmt.Errorf("granting permissions to shared global bucket key: %w", err) + return garage.S3APICredentials{}, fmt.Errorf( + "granting permissions to shared global bucket key: %w", err, + ) } - return nil + return garage.S3APICredentials{ + ID: createKeyRes.ID, + Secret: createKeyRes.Secret, + }, nil } func garageApplyLayout( @@ -257,15 +261,16 @@ func garageApplyLayout( ) type peerLayout struct { + ID string `json:"id"` Capacity int `json:"capacity"` Zone string `json:"zone"` Tags []string `json:"tags"` } { - clusterLayout := map[string]peerLayout{} + clusterLayout := make([]peerLayout, len(allocs)) - for _, alloc := range allocs { + for i, alloc := range allocs { id := bootstrapGarageHostForAlloc(thisHost, alloc).ID @@ -274,25 +279,26 @@ func garageApplyLayout( zone = alloc.Zone } - clusterLayout[id] = peerLayout{ - Capacity: alloc.Capacity, + clusterLayout[i] = peerLayout{ + ID: id, + Capacity: alloc.Capacity * 1_000_000_000, Zone: zone, Tags: []string{}, } } - err := adminClient.Do(ctx, nil, "POST", "/v0/layout", clusterLayout) + err := adminClient.Do(ctx, nil, "POST", "/v1/layout", clusterLayout) if err != nil { return fmt.Errorf("staging layout changes: %w", err) } } var clusterLayout struct { - Version int `json:"version"` - StagedRoleChanges map[string]peerLayout `json:"stagedRoleChanges"` + Version int `json:"version"` + StagedRoleChanges []peerLayout `json:"stagedRoleChanges"` } - if err := adminClient.Do(ctx, &clusterLayout, "GET", "/v0/layout", nil); err != nil { + if err := adminClient.Do(ctx, &clusterLayout, "GET", "/v1/layout", nil); err != nil { return fmt.Errorf("retrieving staged layout change: %w", err) } @@ -306,7 +312,7 @@ func garageApplyLayout( Version: clusterLayout.Version + 1, } - err := adminClient.Do(ctx, nil, "POST", "/v0/layout/apply", applyClusterLayout) + err := adminClient.Do(ctx, nil, "POST", "/v1/layout/apply", applyClusterLayout) if err != nil { return fmt.Errorf("applying new layout (new version:%d): %w", applyClusterLayout.Version, err) } diff --git a/go/garage/admin_client.go b/go/garage/admin_client.go index e01174e..1b28d85 100644 --- a/go/garage/admin_client.go +++ b/go/garage/admin_client.go @@ -139,12 +139,12 @@ func (c *AdminClient) Wait(ctx context.Context) error { } var clusterStatus struct { - KnownNodes map[string]struct { - IsUp bool `json:"is_up"` - } `json:"knownNodes"` + Nodes []struct { + IsUp bool `json:"isUp"` + } `json:"nodes"` } - err := c.Do(ctx, &clusterStatus, "GET", "/v0/status", nil) + err := c.Do(ctx, &clusterStatus, "GET", "/v1/status", nil) if ctxErr := ctx.Err(); ctxErr != nil { return ctxErr @@ -156,14 +156,14 @@ func (c *AdminClient) Wait(ctx context.Context) error { var numUp int - for _, knownNode := range clusterStatus.KnownNodes { - if knownNode.IsUp { + for _, node := range clusterStatus.Nodes { + if node.IsUp { numUp++ } } ctx := mctx.Annotate(ctx, - "numKnownNodes", len(clusterStatus.KnownNodes), + "numNodes", len(clusterStatus.Nodes), "numUp", numUp, ) diff --git a/go/garage/client.go b/go/garage/client.go index bb355f2..f5c7b86 100644 --- a/go/garage/client.go +++ b/go/garage/client.go @@ -35,14 +35,6 @@ type S3APICredentials struct { Secret string } -// NewS3APICredentials returns a new usable instance of S3APICredentials. -func NewS3APICredentials() S3APICredentials { - return S3APICredentials{ - ID: randStr(8), - Secret: randStr(32), - } -} - // NewS3APIClient returns a minio client configured to use the given garage S3 API // endpoint. func NewS3APIClient(addr string, creds S3APICredentials) S3APIClient { diff --git a/nix/garage.nix b/nix/garage.nix index fb4c538..2ebe030 100644 --- a/nix/garage.nix +++ b/nix/garage.nix @@ -1,20 +1,15 @@ rec { - version = "0.8.1"; + version = "1.0.0"; src = builtins.fetchGit { name = "garage-v${version}"; url = "https://git.deuxfleurs.fr/Deuxfleurs/garage.git"; - rev = "76230f20282e73a5a5afa33af68152acaf732cf5"; + rev = "ff093ddbb8485409f389abe7b5e569cb38d222d2"; }; - # TODO compiling garage broke using 24.05, so for now use the pkgs pinned in - # the garage repo. Probably will revisit this when garage gets upgraded - # anyway. - pkgsSrc = (import "${src}/nix/common.nix").pkgsSrc; - package = { - #pkgsSrc, + pkgsSrc, buildSystem, hostSystem, }: let @@ -29,6 +24,16 @@ rec { release = true; git_version = version; + + # subset of the default release features, as defined in: + # https://git.deuxfleurs.fr/Deuxfleurs/garage/src/commit/ff093ddbb8485409f389abe7b5e569cb38d222d2/nix/compile.nix#L171 + features = [ + "garage/bundled-libs" + "garage/lmdb" + "garage/sqlite" + "garage/metrics" + "garage/syslog" + ]; }; in diff --git a/nix/pkgs.nix b/nix/pkgs.nix index 1701289..aff24a6 100644 --- a/nix/pkgs.nix +++ b/nix/pkgs.nix @@ -38,7 +38,9 @@ rec { supportedSystems = [ "x86_64-linux" "aarch64-linux" - #"armv7l-linux-musl" # rpi, I think? + + # rpi, TODO remember why this is disabled, try to re-enable it + #"armv7l-linux-musl" "i686-linux" ]; diff --git a/tests/utils/with-1-data-1-empty-node-cluster.sh b/tests/utils/with-1-data-1-empty-node-cluster.sh index 692ff82..e7dfc1b 100644 --- a/tests/utils/with-1-data-1-empty-node-cluster.sh +++ b/tests/utils/with-1-data-1-empty-node-cluster.sh @@ -40,13 +40,13 @@ if [ ! -d "$XDG_RUNTIME_DIR/isle" ]; then allocations: - data_path: a/data meta_path: a/meta - capacity: 100 + capacity: 1 - data_path: b/data meta_path: b/meta - capacity: 100 + capacity: 1 - data_path: c/data meta_path: c/meta - capacity: 100 + capacity: 1 EOF echo "Creating 1-data-1-empty network"