diff --git a/AppDir/etc/daemon.yml b/AppDir/etc/daemon.yml index 1baf983..2ff913d 100644 --- a/AppDir/etc/daemon.yml +++ b/AppDir/etc/daemon.yml @@ -69,7 +69,7 @@ storage: # the meta directories can be anywhere (ideally on an SSD). # # Capacity declares how many gigabytes can be stored in each allocation, and - # is required. It must be a multiple of 100. + # is required. # # The ports are all _optional_, and will be automatically assigned if they are # not specified. If ports any ports are specified then all should be diff --git a/default.nix b/default.nix index 20d2007..f752824 100644 --- a/default.nix +++ b/default.nix @@ -90,7 +90,7 @@ in rec { inherit buildSystem; hostSystem = "${hostPlatform.cpu.name}-unknown-${hostPlatform.kernel.name}-musl"; - #pkgsSrc = pkgsNix.src; + pkgsSrc = pkgsNix.src; }; diff --git a/docs/operator/contributing-storage.md b/docs/operator/contributing-storage.md index 7633a36..c5e2c0b 100644 --- a/docs/operator/contributing-storage.md +++ b/docs/operator/contributing-storage.md @@ -35,7 +35,7 @@ storage: meta_path: /mnt/drive1/isle/meta capacity: 1200 - # 100 GB (the minimum) are being shared from drive2 + # 100 GB are being shared from drive2 - data_path: /mnt/drive2/isle/data meta_path: /mnt/drive2/isle/meta capacity: 100 diff --git a/go/bootstrap/bootstrap.go b/go/bootstrap/bootstrap.go index 6ebc786..6470c6c 100644 --- a/go/bootstrap/bootstrap.go +++ b/go/bootstrap/bootstrap.go @@ -30,10 +30,13 @@ func AppDirPath(appDirPath string) string { // Garage contains parameters needed to connect to and use the garage cluster. type Garage struct { - // TODO RPCSecret and GlobalBucketS3APICredentials are duplicated here and - // in AdminCreationParams, might as well just use them from there - RPCSecret string - AdminToken string + // TODO this should be part of some new configuration section related to + // secrets which may or may not be granted to this host + RPCSecret string + + AdminToken string + + // TODO this should be part of admin.CreationParams GlobalBucketS3APICredentials garage.S3APICredentials } diff --git a/go/cmd/entrypoint/admin.go b/go/cmd/entrypoint/admin.go index 8e31753..ffeb7e7 100644 --- a/go/cmd/entrypoint/admin.go +++ b/go/cmd/entrypoint/admin.go @@ -149,9 +149,8 @@ var subCmdAdminCreateNetwork = subCmd{ } garageBootstrap := bootstrap.Garage{ - RPCSecret: randStr(32), - AdminToken: randStr(32), - GlobalBucketS3APICredentials: garage.NewS3APICredentials(), + RPCSecret: randStr(32), + AdminToken: randStr(32), } hostBootstrap, err := bootstrap.New( @@ -216,7 +215,17 @@ var subCmdAdminCreateNetwork = subCmd{ } logger.Info(ctx, "initializing garage shared global bucket") - err = garageInitializeGlobalBucket(ctx, logger, hostBootstrap, daemonConfig) + garageGlobalBucketCreds, err := garageInitializeGlobalBucket( + ctx, logger, hostBootstrap, daemonConfig, + ) + + hostBootstrap.Garage.GlobalBucketS3APICredentials = garageGlobalBucketCreds + + // rewrite the bootstrap now that the global bucket creds have been + // added to it. + if err := writeBootstrapToDataDir(hostBootstrap); err != nil { + return fmt.Errorf("writing bootstrap file: %w", err) + } if cErr := (garage.AdminClientError{}); errors.As(err, &cErr) && cErr.StatusCode == 409 { return fmt.Errorf("shared global bucket has already been created, are the storage allocations from a previously initialized isle being used?") diff --git a/go/cmd/entrypoint/garage_util.go b/go/cmd/entrypoint/garage_util.go index 5be8d46..dff0b02 100644 --- a/go/cmd/entrypoint/garage_util.go +++ b/go/cmd/entrypoint/garage_util.go @@ -2,10 +2,10 @@ package main import ( "context" + "fmt" "isle/bootstrap" "isle/daemon" "isle/garage" - "fmt" "net" "path/filepath" "strconv" @@ -184,51 +184,50 @@ func garageInitializeGlobalBucket( logger *mlog.Logger, hostBootstrap bootstrap.Bootstrap, daemonConfig daemon.Config, -) error { +) ( + garage.S3APICredentials, error, +) { + adminClient := newGarageAdminClient(logger, hostBootstrap, daemonConfig) - var ( - adminClient = newGarageAdminClient(logger, hostBootstrap, daemonConfig) - globalBucketCreds = hostBootstrap.Garage.GlobalBucketS3APICredentials - ) + var createKeyRes struct { + ID string `json:"accessKeyId"` + Secret string `json:"secretAccessKey"` + } // first attempt to import the key - err := adminClient.Do(ctx, nil, "POST", "/v0/key/import", map[string]string{ - "accessKeyId": globalBucketCreds.ID, - "secretAccessKey": globalBucketCreds.Secret, - "name": "shared-global-bucket-key", - }) + err := adminClient.Do( + ctx, &createKeyRes, "POST", "/v1/key", map[string]string{ + "name": "shared-global-bucket-key", + }, + ) if err != nil { - return fmt.Errorf("importing global bucket key into garage: %w", err) + return garage.S3APICredentials{}, fmt.Errorf( + "importing global bucket key into garage: %w", err, + ) } // create global bucket - err = adminClient.Do(ctx, nil, "POST", "/v0/bucket", map[string]string{ - "globalAlias": garage.GlobalBucket, - }) - - if err != nil { - return fmt.Errorf("creating global bucket: %w", err) - } - - // retrieve newly created bucket's id - var getBucketRes struct { + var createBucketRes struct { ID string `json:"id"` } err = adminClient.Do( - ctx, &getBucketRes, - "GET", "/v0/bucket?globalAlias="+garage.GlobalBucket, nil, + ctx, &createBucketRes, "POST", "/v1/bucket", map[string]string{ + "globalAlias": garage.GlobalBucket, + }, ) if err != nil { - return fmt.Errorf("fetching global bucket id: %w", err) + return garage.S3APICredentials{}, fmt.Errorf( + "creating global bucket: %w", err, + ) } // allow shared global bucket key to perform all operations - err = adminClient.Do(ctx, nil, "POST", "/v0/bucket/allow", map[string]interface{}{ - "bucketId": getBucketRes.ID, - "accessKeyId": globalBucketCreds.ID, + err = adminClient.Do(ctx, nil, "POST", "/v1/bucket/allow", map[string]interface{}{ + "bucketId": createBucketRes.ID, + "accessKeyId": createKeyRes.ID, "permissions": map[string]bool{ "read": true, "write": true, @@ -236,10 +235,15 @@ func garageInitializeGlobalBucket( }) if err != nil { - return fmt.Errorf("granting permissions to shared global bucket key: %w", err) + return garage.S3APICredentials{}, fmt.Errorf( + "granting permissions to shared global bucket key: %w", err, + ) } - return nil + return garage.S3APICredentials{ + ID: createKeyRes.ID, + Secret: createKeyRes.Secret, + }, nil } func garageApplyLayout( @@ -257,15 +261,16 @@ func garageApplyLayout( ) type peerLayout struct { + ID string `json:"id"` Capacity int `json:"capacity"` Zone string `json:"zone"` Tags []string `json:"tags"` } { - clusterLayout := map[string]peerLayout{} + clusterLayout := make([]peerLayout, len(allocs)) - for _, alloc := range allocs { + for i, alloc := range allocs { id := bootstrapGarageHostForAlloc(thisHost, alloc).ID @@ -274,25 +279,26 @@ func garageApplyLayout( zone = alloc.Zone } - clusterLayout[id] = peerLayout{ - Capacity: alloc.Capacity, + clusterLayout[i] = peerLayout{ + ID: id, + Capacity: alloc.Capacity * 1_000_000_000, Zone: zone, Tags: []string{}, } } - err := adminClient.Do(ctx, nil, "POST", "/v0/layout", clusterLayout) + err := adminClient.Do(ctx, nil, "POST", "/v1/layout", clusterLayout) if err != nil { return fmt.Errorf("staging layout changes: %w", err) } } var clusterLayout struct { - Version int `json:"version"` - StagedRoleChanges map[string]peerLayout `json:"stagedRoleChanges"` + Version int `json:"version"` + StagedRoleChanges []peerLayout `json:"stagedRoleChanges"` } - if err := adminClient.Do(ctx, &clusterLayout, "GET", "/v0/layout", nil); err != nil { + if err := adminClient.Do(ctx, &clusterLayout, "GET", "/v1/layout", nil); err != nil { return fmt.Errorf("retrieving staged layout change: %w", err) } @@ -306,7 +312,7 @@ func garageApplyLayout( Version: clusterLayout.Version + 1, } - err := adminClient.Do(ctx, nil, "POST", "/v0/layout/apply", applyClusterLayout) + err := adminClient.Do(ctx, nil, "POST", "/v1/layout/apply", applyClusterLayout) if err != nil { return fmt.Errorf("applying new layout (new version:%d): %w", applyClusterLayout.Version, err) } diff --git a/go/garage/admin_client.go b/go/garage/admin_client.go index e01174e..1b28d85 100644 --- a/go/garage/admin_client.go +++ b/go/garage/admin_client.go @@ -139,12 +139,12 @@ func (c *AdminClient) Wait(ctx context.Context) error { } var clusterStatus struct { - KnownNodes map[string]struct { - IsUp bool `json:"is_up"` - } `json:"knownNodes"` + Nodes []struct { + IsUp bool `json:"isUp"` + } `json:"nodes"` } - err := c.Do(ctx, &clusterStatus, "GET", "/v0/status", nil) + err := c.Do(ctx, &clusterStatus, "GET", "/v1/status", nil) if ctxErr := ctx.Err(); ctxErr != nil { return ctxErr @@ -156,14 +156,14 @@ func (c *AdminClient) Wait(ctx context.Context) error { var numUp int - for _, knownNode := range clusterStatus.KnownNodes { - if knownNode.IsUp { + for _, node := range clusterStatus.Nodes { + if node.IsUp { numUp++ } } ctx := mctx.Annotate(ctx, - "numKnownNodes", len(clusterStatus.KnownNodes), + "numNodes", len(clusterStatus.Nodes), "numUp", numUp, ) diff --git a/go/garage/client.go b/go/garage/client.go index bb355f2..f5c7b86 100644 --- a/go/garage/client.go +++ b/go/garage/client.go @@ -35,14 +35,6 @@ type S3APICredentials struct { Secret string } -// NewS3APICredentials returns a new usable instance of S3APICredentials. -func NewS3APICredentials() S3APICredentials { - return S3APICredentials{ - ID: randStr(8), - Secret: randStr(32), - } -} - // NewS3APIClient returns a minio client configured to use the given garage S3 API // endpoint. func NewS3APIClient(addr string, creds S3APICredentials) S3APIClient { diff --git a/nix/garage.nix b/nix/garage.nix index fb4c538..2ebe030 100644 --- a/nix/garage.nix +++ b/nix/garage.nix @@ -1,20 +1,15 @@ rec { - version = "0.8.1"; + version = "1.0.0"; src = builtins.fetchGit { name = "garage-v${version}"; url = "https://git.deuxfleurs.fr/Deuxfleurs/garage.git"; - rev = "76230f20282e73a5a5afa33af68152acaf732cf5"; + rev = "ff093ddbb8485409f389abe7b5e569cb38d222d2"; }; - # TODO compiling garage broke using 24.05, so for now use the pkgs pinned in - # the garage repo. Probably will revisit this when garage gets upgraded - # anyway. - pkgsSrc = (import "${src}/nix/common.nix").pkgsSrc; - package = { - #pkgsSrc, + pkgsSrc, buildSystem, hostSystem, }: let @@ -29,6 +24,16 @@ rec { release = true; git_version = version; + + # subset of the default release features, as defined in: + # https://git.deuxfleurs.fr/Deuxfleurs/garage/src/commit/ff093ddbb8485409f389abe7b5e569cb38d222d2/nix/compile.nix#L171 + features = [ + "garage/bundled-libs" + "garage/lmdb" + "garage/sqlite" + "garage/metrics" + "garage/syslog" + ]; }; in diff --git a/nix/pkgs.nix b/nix/pkgs.nix index 1701289..aff24a6 100644 --- a/nix/pkgs.nix +++ b/nix/pkgs.nix @@ -38,7 +38,9 @@ rec { supportedSystems = [ "x86_64-linux" "aarch64-linux" - #"armv7l-linux-musl" # rpi, I think? + + # rpi, TODO remember why this is disabled, try to re-enable it + #"armv7l-linux-musl" "i686-linux" ]; diff --git a/tests/utils/with-1-data-1-empty-node-cluster.sh b/tests/utils/with-1-data-1-empty-node-cluster.sh index 692ff82..e7dfc1b 100644 --- a/tests/utils/with-1-data-1-empty-node-cluster.sh +++ b/tests/utils/with-1-data-1-empty-node-cluster.sh @@ -40,13 +40,13 @@ if [ ! -d "$XDG_RUNTIME_DIR/isle" ]; then allocations: - data_path: a/data meta_path: a/meta - capacity: 100 + capacity: 1 - data_path: b/data meta_path: b/meta - capacity: 100 + capacity: 1 - data_path: c/data meta_path: c/meta - capacity: 100 + capacity: 1 EOF echo "Creating 1-data-1-empty network"