isle/go/daemon/network/garage.go

package network

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"isle/bootstrap"
	"isle/daemon/daecommon"
	"isle/garage"
	"isle/garage/garagesrv"
	"isle/nebula"
	"isle/secrets"
	"isle/toolkit"
	"net/netip"
	"path/filepath"
	"time"

	"dev.mediocregopher.com/mediocre-go-lib.git/mctx"
	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
	"github.com/minio/minio-go/v7"
)

// Paths within garage's global bucket.
const (
	garageGlobalBucketBootstrapHostsDirPath = "bootstrap/hosts"
)

func getGarageClientParams(
	ctx context.Context,
	secretsStore secrets.Store,
	currBootstrap bootstrap.Bootstrap,
) (
	GarageClientParams, error,
) {
	creds, err := daecommon.GetGarageS3APIGlobalBucketCredentials(
		ctx, secretsStore,
	)
	if err != nil {
		return GarageClientParams{}, fmt.Errorf("getting garage global bucket creds: %w", err)
	}

	rpcSecret, err := daecommon.GetGarageRPCSecret(ctx, secretsStore)
	if err != nil && !errors.Is(err, secrets.ErrNotFound) {
		return GarageClientParams{}, fmt.Errorf("getting garage rpc secret: %w", err)
	}

	return GarageClientParams{
		Node:                         currBootstrap.ChooseGarageNode(),
		GlobalBucketS3APICredentials: creds,
		RPCSecret:                    rpcSecret,
	}, nil
}

func garageInitAllocs(
	hostIP netip.Addr,
	allocs []daecommon.ConfigStorageAllocation,
) (
	[]garage.RemoteNode, error,
) {
	peers := make([]garage.RemoteNode, len(allocs))
	for i, alloc := range allocs {
		id, err := garagesrv.InitAlloc(alloc.MetaPath)
		if err != nil {
			return nil, fmt.Errorf("initializing alloc %+v: %w", alloc, err)
		}

		peers[i] = garage.RemoteNode{
			Node: garage.Node{
				IP:        hostIP.String(),
				RPCPort:   alloc.RPCPort,
				S3APIPort: alloc.S3APIPort,
			},
			ID: id,
		}
	}

	return peers, nil
}

func garageAllocsToRoles(
	host bootstrap.Host, allocs []daecommon.ConfigStorageAllocation,
) (
	[]garage.Role, error,
) {
	var (
		hostName = host.Name
		roles    = make([]garage.Role, len(allocs))
	)

	for i, alloc := range allocs {
		id, err := garagesrv.LoadAllocID(alloc.MetaPath)
		if err != nil {
			return nil, fmt.Errorf("getting ID of alloc %+v: %w", alloc, err)
		}

		roles[i] = garage.Role{
			ID:       id,
			Capacity: alloc.Capacity * 1_000_000_000,
			Zone:     string(hostName),
			Tags:     []string{},
		}
	}

	return roles, nil
}

func garageInitializeGlobalBucket(
	ctx context.Context,
	adminClient *garage.AdminClient,
) (
	garage.S3APICredentials, error,
) {
	creds, err := adminClient.CreateS3APICredentials(
		ctx, garage.GlobalBucketS3APICredentialsName,
	)
	if err != nil {
		return creds, fmt.Errorf("creating global bucket credentials: %w", err)
	}

	bucketID, err := adminClient.CreateBucket(ctx, garage.GlobalBucket)
	if err != nil {
		return creds, fmt.Errorf("creating global bucket: %w", err)
	}

	if err := adminClient.GrantBucketPermissions(
		ctx,
		bucketID,
		creds.ID,
		garage.BucketPermissionRead,
		garage.BucketPermissionWrite,
	); err != nil {
		return creds, fmt.Errorf(
			"granting permissions to shared global bucket key: %w", err,
		)
	}

	return creds, nil
}

func getGarageBootstrapHosts(
	ctx context.Context,
	logger *mlog.Logger,
	secretsStore secrets.Store,
	currBootstrap bootstrap.Bootstrap,
) (
	map[nebula.HostName]bootstrap.Host, error,
) {
	garageClientParams, err := getGarageClientParams(
		ctx, secretsStore, currBootstrap,
	)
	if err != nil {
		return nil, fmt.Errorf("getting garage client params: %w", err)
	}

	var (
		client = garageClientParams.GlobalBucketS3APIClient()
		hosts  = map[nebula.HostName]bootstrap.Host{}

		objInfoCh = client.ListObjects(
			ctx, garage.GlobalBucket,
			minio.ListObjectsOptions{
				Prefix:    garageGlobalBucketBootstrapHostsDirPath,
				Recursive: true,
			},
		)
	)

	defer client.Close()

	for objInfo := range objInfoCh {

		ctx := mctx.Annotate(ctx, "objectKey", objInfo.Key)

		if objInfo.Err != nil {
			return nil, fmt.Errorf("listing objects: %w", objInfo.Err)
		}

		obj, err := client.GetObject(
			ctx, garage.GlobalBucket, objInfo.Key, minio.GetObjectOptions{},
		)

		if err != nil {
			return nil, fmt.Errorf("retrieving object %q: %w", objInfo.Key, err)
		}

		var authedHost bootstrap.AuthenticatedHost

		err = json.NewDecoder(obj).Decode(&authedHost)
		obj.Close()

		if err != nil {
			logger.Warn(ctx, "Object contains invalid json", err)
			continue
		}

		host, err := authedHost.Unwrap(currBootstrap.CAPublicCredentials)
		if err != nil {
			logger.Warn(ctx, "Host could not be authenticated", err)
			continue
		}

		hosts[host.Name] = host
	}

	return hosts, nil
}

// putGarageBoostrapHost places the <hostname>.json.signed file for this host
// into garage so that other hosts are able to see relevant configuration for
// it.
func putGarageBoostrapHost(
	ctx context.Context,
	secretsStore secrets.Store,
	currBootstrap bootstrap.Bootstrap,
) error {
	garageClientParams, err := getGarageClientParams(
		ctx, secretsStore, currBootstrap,
	)
	if err != nil {
		return fmt.Errorf("getting garage client params: %w", err)
	}

	var (
		host   = currBootstrap.ThisHost()
		client = garageClientParams.GlobalBucketS3APIClient()
	)

	defer client.Close()

	configured, err := nebula.Sign(
		host.HostConfigured, currBootstrap.PrivateCredentials.SigningPrivateKey,
	)
	if err != nil {
		return fmt.Errorf("signing host configured data: %w", err)
	}

	hostB, err := json.Marshal(bootstrap.AuthenticatedHost{
		Assigned:   currBootstrap.SignedHostAssigned,
		Configured: configured,
	})
	if err != nil {
		return fmt.Errorf("encoding host data: %w", err)
	}

	filePath := filepath.Join(
		garageGlobalBucketBootstrapHostsDirPath,
		string(host.Name)+".json.signed",
	)

	_, err = client.PutObject(
		ctx,
		garage.GlobalBucket,
		filePath,
		bytes.NewReader(hostB),
		int64(len(hostB)),
		minio.PutObjectOptions{},
	)

	if err != nil {
		return fmt.Errorf("writing to %q in global bucket: %w", filePath, err)
	}

	return nil
}

func removeGarageBootstrapHost(
	ctx context.Context, client *garage.S3APIClient, hostName nebula.HostName,
) error {

	filePath := filepath.Join(
		garageGlobalBucketBootstrapHostsDirPath,
		string(hostName)+".json.signed",
	)

	return client.RemoveObject(
		ctx, garage.GlobalBucket, filePath, minio.RemoveObjectOptions{},
	)
}

// We can wait for the garage instance to appear healthy, but there are cases
// where they still haven't fully synced the list of buckets and bucket
// credentials. For those cases it's necessary to do this as an additional
// check.
func garageWaitForAlloc(
	ctx context.Context,
	logger *mlog.Logger,
	adminClient *garage.AdminClient,
) error {
	logger.Info(ctx, "Checking if garage instance has synced bucket list")
	if err := toolkit.UntilTrue(
		ctx,
		logger,
		2*time.Second,
		func() (bool, error) {
			buckets, err := adminClient.ListBuckets(ctx)
			if err != nil {
				return false, fmt.Errorf("listing buckets: %w", err)
			} else if len(buckets) == 0 {
				logger.Info(ctx, "Bucket list not yet synced")
				return false, nil
			}

			return true, nil
		},
	); err != nil {
		return fmt.Errorf("waiting for bucket list to sync: %w", err)
	}

	logger.Info(ctx, "Checking if garage instance has synced credentials")
	if err := toolkit.UntilTrue(
		ctx,
		logger,
		2*time.Second,
		func() (bool, error) {
			credentials, err := adminClient.ListS3APICredentials(ctx)
			if err != nil {
				return false, fmt.Errorf("listing credentials: %w", err)
			} else if len(credentials) == 0 {
				logger.Info(ctx, "Credentials not yet synced")
				return false, nil
			}

			return true, nil
		},
	); err != nil {
		return fmt.Errorf("waiting for credentials list to sync: %w", err)
	}

	return nil
}