micropelago/isle
4
1
Fork 0
Code Issues 1 Pull Requests Projects Releases 5 Activity

Move garage admin API calls into garage package

Browse Source
This commit is contained in:
Brian Picciano 2024-06-12 10:53:06 +02:00
parent 842c169169
commit 65fa208a34
3 changed files with 186 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
go/cmd/entrypoint/garage_util.go
View File

@ -31,12 +31,12 @@ func newGarageAdminClient(
thisHost := hostBootstrap.ThisHost()
return garage.NewAdminClient(
garageAdminClientLogger(logger),
net.JoinHostPort(
thisHost.IP().String(),
strconv.Itoa(daemonConfig.Storage.Allocations[0].AdminPort),
),
hostBootstrap.Garage.AdminToken,
garageAdminClientLogger(logger),
)
}
@ -69,9 +69,9 @@ func waitForGarageAndNebula(
)
adminClient := garage.NewAdminClient(
adminClientLogger,
adminAddr,
hostBootstrap.Garage.AdminToken,
adminClientLogger,
)
ctx := mctx.Annotate(ctx, "garageAdminAddr", adminAddr)
@ -187,61 +187,31 @@ func garageInitializeGlobalBucket(
) {
adminClient := newGarageAdminClient(logger, hostBootstrap, daemonConfig)
var createKeyRes struct {
ID string `json:"accessKeyId"`
Secret string `json:"secretAccessKey"`
}
// first attempt to import the key
err := adminClient.Do(
ctx, &createKeyRes, "POST", "/v1/key", map[string]string{
"name": "shared-global-bucket-key",
},
creds, err := adminClient.CreateS3APICredentials(
ctx, garage.GlobalBucketS3APICredentialsName,
)
if err != nil {
return garage.S3APICredentials{}, fmt.Errorf(
"importing global bucket key into garage: %w", err,
)
return creds, fmt.Errorf("creating global bucket credentials: %w", err)
}
// create global bucket
var createBucketRes struct {
ID string `json:"id"`
bucketID, err := adminClient.CreateBucket(ctx, garage.GlobalBucket)
if err != nil {
return creds, fmt.Errorf("creating global bucket: %w", err)
}
err = adminClient.Do(
ctx, &createBucketRes, "POST", "/v1/bucket", map[string]string{
"globalAlias": garage.GlobalBucket,
},
)
if err != nil {
return garage.S3APICredentials{}, fmt.Errorf(
"creating global bucket: %w", err,
)
}
// allow shared global bucket key to perform all operations
err = adminClient.Do(ctx, nil, "POST", "/v1/bucket/allow", map[string]interface{}{
"bucketId": createBucketRes.ID,
"accessKeyId": createKeyRes.ID,
"permissions": map[string]bool{
"read": true,
"write": true,
},
})
if err != nil {
return garage.S3APICredentials{}, fmt.Errorf(
if err := adminClient.GrantBucketPermissions(
ctx,
bucketID,
creds.ID,
garage.BucketPermissionRead,
garage.BucketPermissionWrite,
); err != nil {
return creds, fmt.Errorf(
"granting permissions to shared global bucket key: %w", err,
)
}
return garage.S3APICredentials{
ID: createKeyRes.ID,
Secret: createKeyRes.Secret,
}, nil
return creds, nil
}
func garageApplyLayout(
@ -256,64 +226,25 @@ func garageApplyLayout(
thisHost = hostBootstrap.ThisHost()
hostName = thisHost.Name
allocs = daemonConfig.Storage.Allocations
peers = make([]garage.PeerLayout, len(allocs))
)
type peerLayout struct {
ID string `json:"id"`
Capacity int `json:"capacity"`
Zone string `json:"zone"`
Tags []string `json:"tags"`
}
for i, alloc := range allocs {
{
clusterLayout := make([]peerLayout, len(allocs))
id := bootstrapGarageHostForAlloc(thisHost, alloc).ID
for i, alloc := range allocs {
id := bootstrapGarageHostForAlloc(thisHost, alloc).ID
zone := hostName
if alloc.Zone != "" {
zone = alloc.Zone
}
clusterLayout[i] = peerLayout{
ID: id,
Capacity: alloc.Capacity * 1_000_000_000,
Zone: zone,
Tags: []string{},
}
zone := hostName
if alloc.Zone != "" {
zone = alloc.Zone
}
err := adminClient.Do(ctx, nil, "POST", "/v1/layout", clusterLayout)
if err != nil {
return fmt.Errorf("staging layout changes: %w", err)
peers[i] = garage.PeerLayout{
ID: id,
Capacity: alloc.Capacity * 1_000_000_000,
Zone: zone,
Tags: []string{},
}
}
var clusterLayout struct {
Version int `json:"version"`
StagedRoleChanges []peerLayout `json:"stagedRoleChanges"`
}
if err := adminClient.Do(ctx, &clusterLayout, "GET", "/v1/layout", nil); err != nil {
return fmt.Errorf("retrieving staged layout change: %w", err)
}
if len(clusterLayout.StagedRoleChanges) == 0 {
return nil
}
applyClusterLayout := struct {
Version int `json:"version"`
}{
Version: clusterLayout.Version + 1,
}
err := adminClient.Do(ctx, nil, "POST", "/v1/layout/apply", applyClusterLayout)
if err != nil {
return fmt.Errorf("applying new layout (new version:%d): %w", applyClusterLayout.Version, err)
}
return nil
return adminClient.ApplyLayout(ctx, peers)
}

164
go/garage/admin_client.go
View File

@ -14,7 +14,22 @@ import (
"github.com/mediocregopher/mediocre-go-lib/v2/mlog"
)
// AdminClientError gets returned from AdminClient's Do method for non-200
// BucketID is a unique identifier for a bucket in garage. It is different than
// the bucket's alias, since a bucket may have multiple aliases.
type BucketID string
// BucketPermission describes a permission an S3APICredentials may have when
// interacting with a bucket.
type BucketPermission string
// Enumeration of BucketPermissions.
const (
BucketPermissionRead BucketPermission = "read"
BucketPermissionWrite BucketPermission = "write"
BucketPermissionOwner BucketPermission = "owner"
)
// AdminClientError gets returned from AdminClient Do methods for non-200
// errors.
type AdminClientError struct {
StatusCode int
@ -28,32 +43,32 @@ func (e AdminClientError) Error() string {
// AdminClient is a helper type for performing actions against the garage admin
// interface.
type AdminClient struct {
logger *mlog.Logger
c *http.Client
addr string
adminToken string
logger *mlog.Logger
}
// NewAdminClient initializes and returns an AdminClient which will use the
// given address and adminToken for all requests made.
//
// If Logger is nil then logs will be suppressed.
func NewAdminClient(addr, adminToken string, logger *mlog.Logger) *AdminClient {
func NewAdminClient(logger *mlog.Logger, addr, adminToken string) *AdminClient {
return &AdminClient{
logger: logger,
c: &http.Client{
Transport: http.DefaultTransport.(*http.Transport).Clone(),
},
addr: addr,
adminToken: adminToken,
logger: logger,
}
}
// Do performs an HTTP request with the given method (GET, POST) and path, and
// do performs an HTTP request with the given method (GET, POST) and path, and
// using the json marshaling of the given body as the request body (unless body
// is nil). It will JSON unmarshal the response into rcv, unless rcv is nil.
func (c *AdminClient) Do(
ctx context.Context, rcv interface{}, method, path string, body interface{},
func (c *AdminClient) do(
ctx context.Context, rcv any, method, path string, body any,
) error {
var bodyR io.Reader
@ -112,7 +127,6 @@ func (c *AdminClient) Do(
}
if rcv == nil {
if _, err := io.Copy(io.Discard, res.Body); err != nil {
return fmt.Errorf("discarding response body: %w", err)
}
@ -138,13 +152,14 @@ func (c *AdminClient) Wait(ctx context.Context) error {
time.Sleep(250 * time.Millisecond)
}
// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Nodes/operation/GetNodes
var clusterStatus struct {
Nodes []struct {
IsUp bool `json:"isUp"`
} `json:"nodes"`
}
err := c.Do(ctx, &clusterStatus, "GET", "/v1/status", nil)
err := c.do(ctx, &clusterStatus, "GET", "/v1/status", nil)
if ctxErr := ctx.Err(); ctxErr != nil {
return ctxErr
@ -175,3 +190,134 @@ func (c *AdminClient) Wait(ctx context.Context) error {
c.logger.Debug(ctx, "instance not online yet, will continue waiting")
}
}
// CreateS3APICredentials creates an S3APICredentials with the given name. The
// credentials must be associated with a bucket via a call to
// GrantBucketPermissions in order to be useful.
func (c *AdminClient) CreateS3APICredentials(
ctx context.Context, name string,
) (
S3APICredentials, error,
) {
// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Key/operation/AddKey
var res struct {
ID string `json:"accessKeyId"`
Secret string `json:"secretAccessKey"`
}
// first attempt to import the key
err := c.do(
ctx, &res, "POST", "/v1/key", map[string]string{"name": name},
)
return S3APICredentials{
ID: res.ID,
Secret: res.Secret,
}, err
}
// CreateBucket creates a bucket with the given global alias, returning its ID.
func (c *AdminClient) CreateBucket(
ctx context.Context, globalAlias string,
) (
BucketID, error,
) {
// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Bucket/operation/PutBucketGlobalAlias
var res struct {
ID string `json:"id"`
}
err := c.do(
ctx, &res, "POST", "/v1/bucket", map[string]string{
"globalAlias": globalAlias,
},
)
return BucketID(res.ID), err
}
// GrantBucketPermissions grants the S3APICredentials with the given ID
// permission(s) to interact with the bucket of the given ID.
func (c *AdminClient) GrantBucketPermissions(
ctx context.Context,
bucketID BucketID,
credentialsID string,
perms ...BucketPermission,
) error {
permsMap := map[BucketPermission]bool{}
for _, perm := range perms {
permsMap[perm] = true
}
// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Bucket/operation/AllowBucketKey
return c.do(ctx, nil, "POST", "/v1/bucket/allow", map[string]any{
"bucketId": bucketID,
"accessKeyId": credentialsID,
"permissions": permsMap,
})
}
// PeerLayout describes the properties of a garage peer in the context of the
// layout of the cluster.
type PeerLayout struct {
ID string
Capacity int // Gb (SI units)
Zone string
Tags []string
}
// ApplyLayout modifies the layout of the garage cluster. Only layout of the
// given peers will be modified/created, other peers are not affected.
func (c *AdminClient) ApplyLayout(
ctx context.Context, peers []PeerLayout,
) error {
type peerLayout struct {
ID string `json:"id"`
Capacity int `json:"capacity"`
Zone string `json:"zone"`
Tags []string `json:"tags"`
}
{
// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Layout/operation/ApplyLayout
clusterLayout := make([]peerLayout, len(peers))
for i := range peers {
clusterLayout[i] = peerLayout(peers[i])
}
err := c.do(ctx, nil, "POST", "/v1/layout", clusterLayout)
if err != nil {
return fmt.Errorf("staging layout changes: %w", err)
}
}
// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Layout/operation/GetLayout
var clusterLayout struct {
Version int `json:"version"`
StagedRoleChanges []peerLayout `json:"stagedRoleChanges"`
}
if err := c.do(ctx, &clusterLayout, "GET", "/v1/layout", nil); err != nil {
return fmt.Errorf("retrieving staged layout change: %w", err)
}
if len(clusterLayout.StagedRoleChanges) == 0 {
return nil
}
// https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html#tag/Layout/operation/ApplyLayout
applyClusterLayout := struct {
Version int `json:"version"`
}{
Version: clusterLayout.Version + 1,
}
err := c.do(ctx, nil, "POST", "/v1/layout/apply", applyClusterLayout)
if err != nil {
return fmt.Errorf("applying new layout (new version:%d): %w", applyClusterLayout.Version, err)
}
return nil
}

2
go/garage/garage.go
View File

@ -11,6 +11,8 @@ const (
// accessible to all hosts in the network.
GlobalBucket = "global-shared"
GlobalBucketS3APICredentialsName = "global-shared-key"
// ReplicationFactor indicates the replication factor set on the garage
// cluster. We currently only support a factor of 3.
ReplicationFactor = 3