Upgrade garage to v1.0.0

This required switching all garage admin API calls to the new v1
versions, and redoing how the global bucket key is created so it is
created via the "create key" API call.
This commit is contained in:
Brian Picciano 2024-06-11 14:54:26 +02:00
parent 2768be00d8
commit 68f417b5ba
11 changed files with 94 additions and 77 deletions

View File

@ -69,7 +69,7 @@ storage:
# the meta directories can be anywhere (ideally on an SSD).
#
# Capacity declares how many gigabytes can be stored in each allocation, and
# is required. It must be a multiple of 100.
# is required.
#
# The ports are all _optional_, and will be automatically assigned if they are
# not specified. If ports any ports are specified then all should be

View File

@ -90,7 +90,7 @@ in rec {
inherit buildSystem;
hostSystem = "${hostPlatform.cpu.name}-unknown-${hostPlatform.kernel.name}-musl";
#pkgsSrc = pkgsNix.src;
pkgsSrc = pkgsNix.src;
};

View File

@ -35,7 +35,7 @@ storage:
meta_path: /mnt/drive1/isle/meta
capacity: 1200
# 100 GB (the minimum) are being shared from drive2
# 100 GB are being shared from drive2
- data_path: /mnt/drive2/isle/data
meta_path: /mnt/drive2/isle/meta
capacity: 100

View File

@ -30,10 +30,13 @@ func AppDirPath(appDirPath string) string {
// Garage contains parameters needed to connect to and use the garage cluster.
type Garage struct {
// TODO RPCSecret and GlobalBucketS3APICredentials are duplicated here and
// in AdminCreationParams, might as well just use them from there
RPCSecret string
AdminToken string
// TODO this should be part of some new configuration section related to
// secrets which may or may not be granted to this host
RPCSecret string
AdminToken string
// TODO this should be part of admin.CreationParams
GlobalBucketS3APICredentials garage.S3APICredentials
}

View File

@ -149,9 +149,8 @@ var subCmdAdminCreateNetwork = subCmd{
}
garageBootstrap := bootstrap.Garage{
RPCSecret: randStr(32),
AdminToken: randStr(32),
GlobalBucketS3APICredentials: garage.NewS3APICredentials(),
RPCSecret: randStr(32),
AdminToken: randStr(32),
}
hostBootstrap, err := bootstrap.New(
@ -216,7 +215,17 @@ var subCmdAdminCreateNetwork = subCmd{
}
logger.Info(ctx, "initializing garage shared global bucket")
err = garageInitializeGlobalBucket(ctx, logger, hostBootstrap, daemonConfig)
garageGlobalBucketCreds, err := garageInitializeGlobalBucket(
ctx, logger, hostBootstrap, daemonConfig,
)
hostBootstrap.Garage.GlobalBucketS3APICredentials = garageGlobalBucketCreds
// rewrite the bootstrap now that the global bucket creds have been
// added to it.
if err := writeBootstrapToDataDir(hostBootstrap); err != nil {
return fmt.Errorf("writing bootstrap file: %w", err)
}
if cErr := (garage.AdminClientError{}); errors.As(err, &cErr) && cErr.StatusCode == 409 {
return fmt.Errorf("shared global bucket has already been created, are the storage allocations from a previously initialized isle being used?")

View File

@ -2,10 +2,10 @@ package main
import (
"context"
"fmt"
"isle/bootstrap"
"isle/daemon"
"isle/garage"
"fmt"
"net"
"path/filepath"
"strconv"
@ -184,51 +184,50 @@ func garageInitializeGlobalBucket(
logger *mlog.Logger,
hostBootstrap bootstrap.Bootstrap,
daemonConfig daemon.Config,
) error {
) (
garage.S3APICredentials, error,
) {
adminClient := newGarageAdminClient(logger, hostBootstrap, daemonConfig)
var (
adminClient = newGarageAdminClient(logger, hostBootstrap, daemonConfig)
globalBucketCreds = hostBootstrap.Garage.GlobalBucketS3APICredentials
)
var createKeyRes struct {
ID string `json:"accessKeyId"`
Secret string `json:"secretAccessKey"`
}
// first attempt to import the key
err := adminClient.Do(ctx, nil, "POST", "/v0/key/import", map[string]string{
"accessKeyId": globalBucketCreds.ID,
"secretAccessKey": globalBucketCreds.Secret,
"name": "shared-global-bucket-key",
})
err := adminClient.Do(
ctx, &createKeyRes, "POST", "/v1/key", map[string]string{
"name": "shared-global-bucket-key",
},
)
if err != nil {
return fmt.Errorf("importing global bucket key into garage: %w", err)
return garage.S3APICredentials{}, fmt.Errorf(
"importing global bucket key into garage: %w", err,
)
}
// create global bucket
err = adminClient.Do(ctx, nil, "POST", "/v0/bucket", map[string]string{
"globalAlias": garage.GlobalBucket,
})
if err != nil {
return fmt.Errorf("creating global bucket: %w", err)
}
// retrieve newly created bucket's id
var getBucketRes struct {
var createBucketRes struct {
ID string `json:"id"`
}
err = adminClient.Do(
ctx, &getBucketRes,
"GET", "/v0/bucket?globalAlias="+garage.GlobalBucket, nil,
ctx, &createBucketRes, "POST", "/v1/bucket", map[string]string{
"globalAlias": garage.GlobalBucket,
},
)
if err != nil {
return fmt.Errorf("fetching global bucket id: %w", err)
return garage.S3APICredentials{}, fmt.Errorf(
"creating global bucket: %w", err,
)
}
// allow shared global bucket key to perform all operations
err = adminClient.Do(ctx, nil, "POST", "/v0/bucket/allow", map[string]interface{}{
"bucketId": getBucketRes.ID,
"accessKeyId": globalBucketCreds.ID,
err = adminClient.Do(ctx, nil, "POST", "/v1/bucket/allow", map[string]interface{}{
"bucketId": createBucketRes.ID,
"accessKeyId": createKeyRes.ID,
"permissions": map[string]bool{
"read": true,
"write": true,
@ -236,10 +235,15 @@ func garageInitializeGlobalBucket(
})
if err != nil {
return fmt.Errorf("granting permissions to shared global bucket key: %w", err)
return garage.S3APICredentials{}, fmt.Errorf(
"granting permissions to shared global bucket key: %w", err,
)
}
return nil
return garage.S3APICredentials{
ID: createKeyRes.ID,
Secret: createKeyRes.Secret,
}, nil
}
func garageApplyLayout(
@ -257,15 +261,16 @@ func garageApplyLayout(
)
type peerLayout struct {
ID string `json:"id"`
Capacity int `json:"capacity"`
Zone string `json:"zone"`
Tags []string `json:"tags"`
}
{
clusterLayout := map[string]peerLayout{}
clusterLayout := make([]peerLayout, len(allocs))
for _, alloc := range allocs {
for i, alloc := range allocs {
id := bootstrapGarageHostForAlloc(thisHost, alloc).ID
@ -274,25 +279,26 @@ func garageApplyLayout(
zone = alloc.Zone
}
clusterLayout[id] = peerLayout{
Capacity: alloc.Capacity,
clusterLayout[i] = peerLayout{
ID: id,
Capacity: alloc.Capacity * 1_000_000_000,
Zone: zone,
Tags: []string{},
}
}
err := adminClient.Do(ctx, nil, "POST", "/v0/layout", clusterLayout)
err := adminClient.Do(ctx, nil, "POST", "/v1/layout", clusterLayout)
if err != nil {
return fmt.Errorf("staging layout changes: %w", err)
}
}
var clusterLayout struct {
Version int `json:"version"`
StagedRoleChanges map[string]peerLayout `json:"stagedRoleChanges"`
Version int `json:"version"`
StagedRoleChanges []peerLayout `json:"stagedRoleChanges"`
}
if err := adminClient.Do(ctx, &clusterLayout, "GET", "/v0/layout", nil); err != nil {
if err := adminClient.Do(ctx, &clusterLayout, "GET", "/v1/layout", nil); err != nil {
return fmt.Errorf("retrieving staged layout change: %w", err)
}
@ -306,7 +312,7 @@ func garageApplyLayout(
Version: clusterLayout.Version + 1,
}
err := adminClient.Do(ctx, nil, "POST", "/v0/layout/apply", applyClusterLayout)
err := adminClient.Do(ctx, nil, "POST", "/v1/layout/apply", applyClusterLayout)
if err != nil {
return fmt.Errorf("applying new layout (new version:%d): %w", applyClusterLayout.Version, err)
}

View File

@ -139,12 +139,12 @@ func (c *AdminClient) Wait(ctx context.Context) error {
}
var clusterStatus struct {
KnownNodes map[string]struct {
IsUp bool `json:"is_up"`
} `json:"knownNodes"`
Nodes []struct {
IsUp bool `json:"isUp"`
} `json:"nodes"`
}
err := c.Do(ctx, &clusterStatus, "GET", "/v0/status", nil)
err := c.Do(ctx, &clusterStatus, "GET", "/v1/status", nil)
if ctxErr := ctx.Err(); ctxErr != nil {
return ctxErr
@ -156,14 +156,14 @@ func (c *AdminClient) Wait(ctx context.Context) error {
var numUp int
for _, knownNode := range clusterStatus.KnownNodes {
if knownNode.IsUp {
for _, node := range clusterStatus.Nodes {
if node.IsUp {
numUp++
}
}
ctx := mctx.Annotate(ctx,
"numKnownNodes", len(clusterStatus.KnownNodes),
"numNodes", len(clusterStatus.Nodes),
"numUp", numUp,
)

View File

@ -35,14 +35,6 @@ type S3APICredentials struct {
Secret string
}
// NewS3APICredentials returns a new usable instance of S3APICredentials.
func NewS3APICredentials() S3APICredentials {
return S3APICredentials{
ID: randStr(8),
Secret: randStr(32),
}
}
// NewS3APIClient returns a minio client configured to use the given garage S3 API
// endpoint.
func NewS3APIClient(addr string, creds S3APICredentials) S3APIClient {

View File

@ -1,20 +1,15 @@
rec {
version = "0.8.1";
version = "1.0.0";
src = builtins.fetchGit {
name = "garage-v${version}";
url = "https://git.deuxfleurs.fr/Deuxfleurs/garage.git";
rev = "76230f20282e73a5a5afa33af68152acaf732cf5";
rev = "ff093ddbb8485409f389abe7b5e569cb38d222d2";
};
# TODO compiling garage broke using 24.05, so for now use the pkgs pinned in
# the garage repo. Probably will revisit this when garage gets upgraded
# anyway.
pkgsSrc = (import "${src}/nix/common.nix").pkgsSrc;
package = {
#pkgsSrc,
pkgsSrc,
buildSystem,
hostSystem,
}: let
@ -29,6 +24,16 @@ rec {
release = true;
git_version = version;
# subset of the default release features, as defined in:
# https://git.deuxfleurs.fr/Deuxfleurs/garage/src/commit/ff093ddbb8485409f389abe7b5e569cb38d222d2/nix/compile.nix#L171
features = [
"garage/bundled-libs"
"garage/lmdb"
"garage/sqlite"
"garage/metrics"
"garage/syslog"
];
};
in

View File

@ -38,7 +38,9 @@ rec {
supportedSystems = [
"x86_64-linux"
"aarch64-linux"
#"armv7l-linux-musl" # rpi, I think?
# rpi, TODO remember why this is disabled, try to re-enable it
#"armv7l-linux-musl"
"i686-linux"
];

View File

@ -40,13 +40,13 @@ if [ ! -d "$XDG_RUNTIME_DIR/isle" ]; then
allocations:
- data_path: a/data
meta_path: a/meta
capacity: 100
capacity: 1
- data_path: b/data
meta_path: b/meta
capacity: 100
capacity: 1
- data_path: c/data
meta_path: c/meta
capacity: 100
capacity: 1
EOF
echo "Creating 1-data-1-empty network"