From 65fa208a34a2ce19a0fcc2037b48e10a0335ad1d Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Wed, 12 Jun 2024 10:53:06 +0200 Subject: [PATCH] Move garage admin API calls into garage package --- go/cmd/entrypoint/garage_util.go | 127 ++++++------------------ go/garage/admin_client.go | 164 +++++++++++++++++++++++++++++-- go/garage/garage.go | 2 + 3 files changed, 186 insertions(+), 107 deletions(-) diff --git a/go/cmd/entrypoint/garage_util.go b/go/cmd/entrypoint/garage_util.go index 792c1d4..a96b688 100644 --- a/go/cmd/entrypoint/garage_util.go +++ b/go/cmd/entrypoint/garage_util.go @@ -31,12 +31,12 @@ func newGarageAdminClient( thisHost := hostBootstrap.ThisHost() return garage.NewAdminClient( + garageAdminClientLogger(logger), net.JoinHostPort( thisHost.IP().String(), strconv.Itoa(daemonConfig.Storage.Allocations[0].AdminPort), ), hostBootstrap.Garage.AdminToken, - garageAdminClientLogger(logger), ) } @@ -69,9 +69,9 @@ func waitForGarageAndNebula( ) adminClient := garage.NewAdminClient( + adminClientLogger, adminAddr, hostBootstrap.Garage.AdminToken, - adminClientLogger, ) ctx := mctx.Annotate(ctx, "garageAdminAddr", adminAddr) @@ -187,61 +187,31 @@ func garageInitializeGlobalBucket( ) { adminClient := newGarageAdminClient(logger, hostBootstrap, daemonConfig) - var createKeyRes struct { - ID string `json:"accessKeyId"` - Secret string `json:"secretAccessKey"` - } - - // first attempt to import the key - err := adminClient.Do( - ctx, &createKeyRes, "POST", "/v1/key", map[string]string{ - "name": "shared-global-bucket-key", - }, + creds, err := adminClient.CreateS3APICredentials( + ctx, garage.GlobalBucketS3APICredentialsName, ) - if err != nil { - return garage.S3APICredentials{}, fmt.Errorf( - "importing global bucket key into garage: %w", err, - ) + return creds, fmt.Errorf("creating global bucket credentials: %w", err) } - // create global bucket - var createBucketRes struct { - ID string `json:"id"` + bucketID, err := adminClient.CreateBucket(ctx, garage.GlobalBucket) + if err != nil { + return creds, fmt.Errorf("creating global bucket: %w", err) } - err = adminClient.Do( - ctx, &createBucketRes, "POST", "/v1/bucket", map[string]string{ - "globalAlias": garage.GlobalBucket, - }, - ) - - if err != nil { - return garage.S3APICredentials{}, fmt.Errorf( - "creating global bucket: %w", err, - ) - } - - // allow shared global bucket key to perform all operations - err = adminClient.Do(ctx, nil, "POST", "/v1/bucket/allow", map[string]interface{}{ - "bucketId": createBucketRes.ID, - "accessKeyId": createKeyRes.ID, - "permissions": map[string]bool{ - "read": true, - "write": true, - }, - }) - - if err != nil { - return garage.S3APICredentials{}, fmt.Errorf( + if err := adminClient.GrantBucketPermissions( + ctx, + bucketID, + creds.ID, + garage.BucketPermissionRead, + garage.BucketPermissionWrite, + ); err != nil { + return creds, fmt.Errorf( "granting permissions to shared global bucket key: %w", err, ) } - return garage.S3APICredentials{ - ID: createKeyRes.ID, - Secret: createKeyRes.Secret, - }, nil + return creds, nil } func garageApplyLayout( @@ -256,64 +226,25 @@ func garageApplyLayout( thisHost = hostBootstrap.ThisHost() hostName = thisHost.Name allocs = daemonConfig.Storage.Allocations + peers = make([]garage.PeerLayout, len(allocs)) ) - type peerLayout struct { - ID string `json:"id"` - Capacity int `json:"capacity"` - Zone string `json:"zone"` - Tags []string `json:"tags"` - } + for i, alloc := range allocs { - { - clusterLayout := make([]peerLayout, len(allocs)) + id := bootstrapGarageHostForAlloc(thisHost, alloc).ID - for i, alloc := range allocs { - - id := bootstrapGarageHostForAlloc(thisHost, alloc).ID - - zone := hostName - if alloc.Zone != "" { - zone = alloc.Zone - } - - clusterLayout[i] = peerLayout{ - ID: id, - Capacity: alloc.Capacity * 1_000_000_000, - Zone: zone, - Tags: []string{}, - } + zone := hostName + if alloc.Zone != "" { + zone = alloc.Zone } - err := adminClient.Do(ctx, nil, "POST", "/v1/layout", clusterLayout) - if err != nil { - return fmt.Errorf("staging layout changes: %w", err) + peers[i] = garage.PeerLayout{ + ID: id, + Capacity: alloc.Capacity * 1_000_000_000, + Zone: zone, + Tags: []string{}, } } - var clusterLayout struct { - Version int `json:"version"` - StagedRoleChanges []peerLayout `json:"stagedRoleChanges"` - } - - if err := adminClient.Do(ctx, &clusterLayout, "GET", "/v1/layout", nil); err != nil { - return fmt.Errorf("retrieving staged layout change: %w", err) - } - - if len(clusterLayout.StagedRoleChanges) == 0 { - return nil - } - - applyClusterLayout := struct { - Version int `json:"version"` - }{ - Version: clusterLayout.Version + 1, - } - - err := adminClient.Do(ctx, nil, "POST", "/v1/layout/apply", applyClusterLayout) - if err != nil { - return fmt.Errorf("applying new layout (new version:%d): %w", applyClusterLayout.Version, err) - } - - return nil + return adminClient.ApplyLayout(ctx, peers) } diff --git a/go/garage/admin_client.go b/go/garage/admin_client.go index 1b28d85..f5b5f21 100644 --- a/go/garage/admin_client.go +++ b/go/garage/admin_client.go @@ -14,7 +14,22 @@ import ( "github.com/mediocregopher/mediocre-go-lib/v2/mlog" ) -// AdminClientError gets returned from AdminClient's Do method for non-200 +// BucketID is a unique identifier for a bucket in garage. It is different than +// the bucket's alias, since a bucket may have multiple aliases. +type BucketID string + +// BucketPermission describes a permission an S3APICredentials may have when +// interacting with a bucket. +type BucketPermission string + +// Enumeration of BucketPermissions. +const ( + BucketPermissionRead BucketPermission = "read" + BucketPermissionWrite BucketPermission = "write" + BucketPermissionOwner BucketPermission = "owner" +) + +// AdminClientError gets returned from AdminClient Do methods for non-200 // errors. type AdminClientError struct { StatusCode int @@ -28,32 +43,32 @@ func (e AdminClientError) Error() string { // AdminClient is a helper type for performing actions against the garage admin // interface. type AdminClient struct { + logger *mlog.Logger c *http.Client addr string adminToken string - logger *mlog.Logger } // NewAdminClient initializes and returns an AdminClient which will use the // given address and adminToken for all requests made. // // If Logger is nil then logs will be suppressed. -func NewAdminClient(addr, adminToken string, logger *mlog.Logger) *AdminClient { +func NewAdminClient(logger *mlog.Logger, addr, adminToken string) *AdminClient { return &AdminClient{ + logger: logger, c: &http.Client{ Transport: http.DefaultTransport.(*http.Transport).Clone(), }, addr: addr, adminToken: adminToken, - logger: logger, } } -// Do performs an HTTP request with the given method (GET, POST) and path, and +// do performs an HTTP request with the given method (GET, POST) and path, and // using the json marshaling of the given body as the request body (unless body // is nil). It will JSON unmarshal the response into rcv, unless rcv is nil. -func (c *AdminClient) Do( - ctx context.Context, rcv interface{}, method, path string, body interface{}, +func (c *AdminClient) do( + ctx context.Context, rcv any, method, path string, body any, ) error { var bodyR io.Reader @@ -112,7 +127,6 @@ func (c *AdminClient) Do( } if rcv == nil { - if _, err := io.Copy(io.Discard, res.Body); err != nil { return fmt.Errorf("discarding response body: %w", err) } @@ -138,13 +152,14 @@ func (c *AdminClient) Wait(ctx context.Context) error { time.Sleep(250 * time.Millisecond) } + // https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Nodes/operation/GetNodes var clusterStatus struct { Nodes []struct { IsUp bool `json:"isUp"` } `json:"nodes"` } - err := c.Do(ctx, &clusterStatus, "GET", "/v1/status", nil) + err := c.do(ctx, &clusterStatus, "GET", "/v1/status", nil) if ctxErr := ctx.Err(); ctxErr != nil { return ctxErr @@ -175,3 +190,134 @@ func (c *AdminClient) Wait(ctx context.Context) error { c.logger.Debug(ctx, "instance not online yet, will continue waiting") } } + +// CreateS3APICredentials creates an S3APICredentials with the given name. The +// credentials must be associated with a bucket via a call to +// GrantBucketPermissions in order to be useful. +func (c *AdminClient) CreateS3APICredentials( + ctx context.Context, name string, +) ( + S3APICredentials, error, +) { + // https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Key/operation/AddKey + + var res struct { + ID string `json:"accessKeyId"` + Secret string `json:"secretAccessKey"` + } + + // first attempt to import the key + err := c.do( + ctx, &res, "POST", "/v1/key", map[string]string{"name": name}, + ) + + return S3APICredentials{ + ID: res.ID, + Secret: res.Secret, + }, err +} + +// CreateBucket creates a bucket with the given global alias, returning its ID. +func (c *AdminClient) CreateBucket( + ctx context.Context, globalAlias string, +) ( + BucketID, error, +) { + // https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Bucket/operation/PutBucketGlobalAlias + + var res struct { + ID string `json:"id"` + } + + err := c.do( + ctx, &res, "POST", "/v1/bucket", map[string]string{ + "globalAlias": globalAlias, + }, + ) + + return BucketID(res.ID), err +} + +// GrantBucketPermissions grants the S3APICredentials with the given ID +// permission(s) to interact with the bucket of the given ID. +func (c *AdminClient) GrantBucketPermissions( + ctx context.Context, + bucketID BucketID, + credentialsID string, + perms ...BucketPermission, +) error { + permsMap := map[BucketPermission]bool{} + for _, perm := range perms { + permsMap[perm] = true + } + + // https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Bucket/operation/AllowBucketKey + return c.do(ctx, nil, "POST", "/v1/bucket/allow", map[string]any{ + "bucketId": bucketID, + "accessKeyId": credentialsID, + "permissions": permsMap, + }) +} + +// PeerLayout describes the properties of a garage peer in the context of the +// layout of the cluster. +type PeerLayout struct { + ID string + Capacity int // Gb (SI units) + Zone string + Tags []string +} + +// ApplyLayout modifies the layout of the garage cluster. Only layout of the +// given peers will be modified/created, other peers are not affected. +func (c *AdminClient) ApplyLayout( + ctx context.Context, peers []PeerLayout, +) error { + type peerLayout struct { + ID string `json:"id"` + Capacity int `json:"capacity"` + Zone string `json:"zone"` + Tags []string `json:"tags"` + } + + { + // https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Layout/operation/ApplyLayout + clusterLayout := make([]peerLayout, len(peers)) + for i := range peers { + clusterLayout[i] = peerLayout(peers[i]) + } + + err := c.do(ctx, nil, "POST", "/v1/layout", clusterLayout) + if err != nil { + return fmt.Errorf("staging layout changes: %w", err) + } + } + + // https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Layout/operation/GetLayout + var clusterLayout struct { + Version int `json:"version"` + StagedRoleChanges []peerLayout `json:"stagedRoleChanges"` + } + + if err := c.do(ctx, &clusterLayout, "GET", "/v1/layout", nil); err != nil { + return fmt.Errorf("retrieving staged layout change: %w", err) + } + + if len(clusterLayout.StagedRoleChanges) == 0 { + return nil + } + + // https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Layout/operation/ApplyLayout + applyClusterLayout := struct { + Version int `json:"version"` + }{ + Version: clusterLayout.Version + 1, + } + + err := c.do(ctx, nil, "POST", "/v1/layout/apply", applyClusterLayout) + if err != nil { + return fmt.Errorf("applying new layout (new version:%d): %w", applyClusterLayout.Version, err) + } + + return nil +} diff --git a/go/garage/garage.go b/go/garage/garage.go index 65095a0..729a451 100644 --- a/go/garage/garage.go +++ b/go/garage/garage.go @@ -11,6 +11,8 @@ const ( // accessible to all hosts in the network. GlobalBucket = "global-shared" + GlobalBucketS3APICredentialsName = "global-shared-key" + // ReplicationFactor indicates the replication factor set on the garage // cluster. We currently only support a factor of 3. ReplicationFactor = 3