Brian Picciano
8c3e6a2845
In a world where the daemon can manage more than one network, the Daemon is really responsible only for knowing which networks are currently joined, creating/joining/leaving networks, and routing incoming RPC requests to the correct network handler as needed. The new network package, with its Network interface, inherits most of the logic that Daemon used to have, leaving Daemon only the parts needed for the functionality just described. There's a lot of cleanup done here in order to really nail down the separation of concerns between the two, especially around directory creation.
280 lines
6.5 KiB
Go
280 lines
6.5 KiB
Go
package network
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"isle/bootstrap"
|
|
"isle/daemon/daecommon"
|
|
"isle/garage"
|
|
"isle/nebula"
|
|
"isle/secrets"
|
|
"net"
|
|
"path/filepath"
|
|
"strconv"
|
|
|
|
"dev.mediocregopher.com/mediocre-go-lib.git/mctx"
|
|
"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
|
|
"github.com/minio/minio-go/v7"
|
|
)
|
|
|
|
// Paths within garage's global bucket.
|
|
const (
|
|
garageGlobalBucketBootstrapHostsDirPath = "bootstrap/hosts"
|
|
)
|
|
|
|
func (n *network) getGarageClientParams(
|
|
ctx context.Context, currBootstrap bootstrap.Bootstrap,
|
|
) (
|
|
GarageClientParams, error,
|
|
) {
|
|
creds, err := daecommon.GetGarageS3APIGlobalBucketCredentials(
|
|
ctx, n.secretsStore,
|
|
)
|
|
if err != nil {
|
|
return GarageClientParams{}, fmt.Errorf("getting garage global bucket creds: %w", err)
|
|
}
|
|
|
|
rpcSecret, err := daecommon.GetGarageRPCSecret(ctx, n.secretsStore)
|
|
if err != nil && !errors.Is(err, secrets.ErrNotFound) {
|
|
return GarageClientParams{}, fmt.Errorf("getting garage rpc secret: %w", err)
|
|
}
|
|
|
|
return GarageClientParams{
|
|
Peer: currBootstrap.ChooseGaragePeer(),
|
|
GlobalBucketS3APICredentials: creds,
|
|
RPCSecret: rpcSecret,
|
|
}, nil
|
|
}
|
|
|
|
func garageAdminClientLogger(logger *mlog.Logger) *mlog.Logger {
|
|
return logger.WithNamespace("garageAdminClient")
|
|
}
|
|
|
|
// newGarageAdminClient will return an AdminClient for a local garage instance,
|
|
// or it will _panic_ if there is no local instance configured.
|
|
func newGarageAdminClient(
|
|
logger *mlog.Logger,
|
|
daemonConfig daecommon.Config,
|
|
adminToken string,
|
|
hostBootstrap bootstrap.Bootstrap,
|
|
) *garage.AdminClient {
|
|
|
|
thisHost := hostBootstrap.ThisHost()
|
|
|
|
return garage.NewAdminClient(
|
|
garageAdminClientLogger(logger),
|
|
net.JoinHostPort(
|
|
thisHost.IP().String(),
|
|
strconv.Itoa(daemonConfig.Storage.Allocations[0].AdminPort),
|
|
),
|
|
adminToken,
|
|
)
|
|
}
|
|
|
|
func garageApplyLayout(
|
|
ctx context.Context,
|
|
logger *mlog.Logger,
|
|
daemonConfig daecommon.Config,
|
|
adminToken string,
|
|
hostBootstrap bootstrap.Bootstrap,
|
|
) error {
|
|
|
|
var (
|
|
adminClient = newGarageAdminClient(
|
|
logger, daemonConfig, adminToken, hostBootstrap,
|
|
)
|
|
thisHost = hostBootstrap.ThisHost()
|
|
hostName = thisHost.Name
|
|
allocs = daemonConfig.Storage.Allocations
|
|
peers = make([]garage.PeerLayout, len(allocs))
|
|
)
|
|
|
|
for i, alloc := range allocs {
|
|
|
|
id := daecommon.BootstrapGarageHostForAlloc(thisHost, alloc).ID
|
|
|
|
zone := string(hostName)
|
|
if alloc.Zone != "" {
|
|
zone = alloc.Zone
|
|
}
|
|
|
|
peers[i] = garage.PeerLayout{
|
|
ID: id,
|
|
Capacity: alloc.Capacity * 1_000_000_000,
|
|
Zone: zone,
|
|
Tags: []string{},
|
|
}
|
|
}
|
|
|
|
return adminClient.ApplyLayout(ctx, peers)
|
|
}
|
|
|
|
func garageInitializeGlobalBucket(
|
|
ctx context.Context,
|
|
logger *mlog.Logger,
|
|
daemonConfig daecommon.Config,
|
|
adminToken string,
|
|
hostBootstrap bootstrap.Bootstrap,
|
|
) (
|
|
garage.S3APICredentials, error,
|
|
) {
|
|
adminClient := newGarageAdminClient(
|
|
logger, daemonConfig, adminToken, hostBootstrap,
|
|
)
|
|
|
|
creds, err := adminClient.CreateS3APICredentials(
|
|
ctx, garage.GlobalBucketS3APICredentialsName,
|
|
)
|
|
if err != nil {
|
|
return creds, fmt.Errorf("creating global bucket credentials: %w", err)
|
|
}
|
|
|
|
bucketID, err := adminClient.CreateBucket(ctx, garage.GlobalBucket)
|
|
if err != nil {
|
|
return creds, fmt.Errorf("creating global bucket: %w", err)
|
|
}
|
|
|
|
if err := adminClient.GrantBucketPermissions(
|
|
ctx,
|
|
bucketID,
|
|
creds.ID,
|
|
garage.BucketPermissionRead,
|
|
garage.BucketPermissionWrite,
|
|
); err != nil {
|
|
return creds, fmt.Errorf(
|
|
"granting permissions to shared global bucket key: %w", err,
|
|
)
|
|
}
|
|
|
|
return creds, nil
|
|
}
|
|
|
|
func (n *network) getGarageBootstrapHosts(
|
|
ctx context.Context, currBootstrap bootstrap.Bootstrap,
|
|
) (
|
|
map[nebula.HostName]bootstrap.Host, error,
|
|
) {
|
|
garageClientParams, err := n.getGarageClientParams(ctx, currBootstrap)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("getting garage client params: %w", err)
|
|
}
|
|
|
|
var (
|
|
client = garageClientParams.GlobalBucketS3APIClient()
|
|
hosts = map[nebula.HostName]bootstrap.Host{}
|
|
|
|
objInfoCh = client.ListObjects(
|
|
ctx, garage.GlobalBucket,
|
|
minio.ListObjectsOptions{
|
|
Prefix: garageGlobalBucketBootstrapHostsDirPath,
|
|
Recursive: true,
|
|
},
|
|
)
|
|
)
|
|
|
|
for objInfo := range objInfoCh {
|
|
|
|
ctx := mctx.Annotate(ctx, "objectKey", objInfo.Key)
|
|
|
|
if objInfo.Err != nil {
|
|
return nil, fmt.Errorf("listing objects: %w", objInfo.Err)
|
|
}
|
|
|
|
obj, err := client.GetObject(
|
|
ctx, garage.GlobalBucket, objInfo.Key, minio.GetObjectOptions{},
|
|
)
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("retrieving object %q: %w", objInfo.Key, err)
|
|
}
|
|
|
|
var authedHost bootstrap.AuthenticatedHost
|
|
|
|
err = json.NewDecoder(obj).Decode(&authedHost)
|
|
obj.Close()
|
|
|
|
if err != nil {
|
|
n.logger.Warn(ctx, "Object contains invalid json", err)
|
|
continue
|
|
}
|
|
|
|
host, err := authedHost.Unwrap(currBootstrap.CAPublicCredentials)
|
|
if err != nil {
|
|
n.logger.Warn(ctx, "Host could not be authenticated", err)
|
|
}
|
|
|
|
hosts[host.Name] = host
|
|
}
|
|
|
|
return hosts, nil
|
|
}
|
|
|
|
// putGarageBoostrapHost places the <hostname>.json.signed file for this host
|
|
// into garage so that other hosts are able to see relevant configuration for
|
|
// it.
|
|
func (n *network) putGarageBoostrapHost(
|
|
ctx context.Context, currBootstrap bootstrap.Bootstrap,
|
|
) error {
|
|
garageClientParams, err := n.getGarageClientParams(ctx, currBootstrap)
|
|
if err != nil {
|
|
return fmt.Errorf("getting garage client params: %w", err)
|
|
}
|
|
|
|
var (
|
|
host = currBootstrap.ThisHost()
|
|
client = garageClientParams.GlobalBucketS3APIClient()
|
|
)
|
|
|
|
configured, err := nebula.Sign(
|
|
host.HostConfigured, currBootstrap.PrivateCredentials.SigningPrivateKey,
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("signing host configured data: %w", err)
|
|
}
|
|
|
|
hostB, err := json.Marshal(bootstrap.AuthenticatedHost{
|
|
Assigned: currBootstrap.SignedHostAssigned,
|
|
Configured: configured,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("encoding host data: %w", err)
|
|
}
|
|
|
|
filePath := filepath.Join(
|
|
garageGlobalBucketBootstrapHostsDirPath,
|
|
string(host.Name)+".json.signed",
|
|
)
|
|
|
|
_, err = client.PutObject(
|
|
ctx,
|
|
garage.GlobalBucket,
|
|
filePath,
|
|
bytes.NewReader(hostB),
|
|
int64(len(hostB)),
|
|
minio.PutObjectOptions{},
|
|
)
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("writing to %q in global bucket: %w", filePath, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func removeGarageBootstrapHost(
|
|
ctx context.Context, client garage.S3APIClient, hostName nebula.HostName,
|
|
) error {
|
|
|
|
filePath := filepath.Join(
|
|
garageGlobalBucketBootstrapHostsDirPath,
|
|
string(hostName)+".json.signed",
|
|
)
|
|
|
|
return client.RemoveObject(
|
|
ctx, garage.GlobalBucket, filePath, minio.RemoveObjectOptions{},
|
|
)
|
|
}
|