package network import ( "bytes" "context" "encoding/json" "errors" "fmt" "isle/bootstrap" "isle/daemon/daecommon" "isle/garage" "isle/nebula" "isle/secrets" "net" "path/filepath" "strconv" "dev.mediocregopher.com/mediocre-go-lib.git/mctx" "dev.mediocregopher.com/mediocre-go-lib.git/mlog" "github.com/minio/minio-go/v7" ) // Paths within garage's global bucket. const ( garageGlobalBucketBootstrapHostsDirPath = "bootstrap/hosts" ) func (n *network) getGarageClientParams( ctx context.Context, currBootstrap bootstrap.Bootstrap, ) ( GarageClientParams, error, ) { creds, err := daecommon.GetGarageS3APIGlobalBucketCredentials( ctx, n.secretsStore, ) if err != nil { return GarageClientParams{}, fmt.Errorf("getting garage global bucket creds: %w", err) } rpcSecret, err := daecommon.GetGarageRPCSecret(ctx, n.secretsStore) if err != nil && !errors.Is(err, secrets.ErrNotFound) { return GarageClientParams{}, fmt.Errorf("getting garage rpc secret: %w", err) } return GarageClientParams{ Peer: currBootstrap.ChooseGaragePeer(), GlobalBucketS3APICredentials: creds, RPCSecret: rpcSecret, }, nil } func garageAdminClientLogger(logger *mlog.Logger) *mlog.Logger { return logger.WithNamespace("garageAdminClient") } // newGarageAdminClient will return an AdminClient for a local garage instance, // or it will _panic_ if there is no local instance configured. func newGarageAdminClient( logger *mlog.Logger, networkConfig daecommon.NetworkConfig, adminToken string, hostBootstrap bootstrap.Bootstrap, ) *garage.AdminClient { thisHost := hostBootstrap.ThisHost() return garage.NewAdminClient( garageAdminClientLogger(logger), net.JoinHostPort( thisHost.IP().String(), strconv.Itoa(networkConfig.Storage.Allocations[0].AdminPort), ), adminToken, ) } func garageApplyLayout( ctx context.Context, logger *mlog.Logger, networkConfig daecommon.NetworkConfig, adminToken string, hostBootstrap bootstrap.Bootstrap, ) error { var ( adminClient = newGarageAdminClient( logger, networkConfig, adminToken, hostBootstrap, ) thisHost = hostBootstrap.ThisHost() hostName = thisHost.Name allocs = networkConfig.Storage.Allocations peers = make([]garage.PeerLayout, len(allocs)) ) for i, alloc := range allocs { id := daecommon.BootstrapGarageHostForAlloc(thisHost, alloc).ID zone := string(hostName) if alloc.Zone != "" { zone = alloc.Zone } peers[i] = garage.PeerLayout{ ID: id, Capacity: alloc.Capacity * 1_000_000_000, Zone: zone, Tags: []string{}, } } return adminClient.ApplyLayout(ctx, peers) } func garageInitializeGlobalBucket( ctx context.Context, logger *mlog.Logger, networkConfig daecommon.NetworkConfig, adminToken string, hostBootstrap bootstrap.Bootstrap, ) ( garage.S3APICredentials, error, ) { adminClient := newGarageAdminClient( logger, networkConfig, adminToken, hostBootstrap, ) creds, err := adminClient.CreateS3APICredentials( ctx, garage.GlobalBucketS3APICredentialsName, ) if err != nil { return creds, fmt.Errorf("creating global bucket credentials: %w", err) } bucketID, err := adminClient.CreateBucket(ctx, garage.GlobalBucket) if err != nil { return creds, fmt.Errorf("creating global bucket: %w", err) } if err := adminClient.GrantBucketPermissions( ctx, bucketID, creds.ID, garage.BucketPermissionRead, garage.BucketPermissionWrite, ); err != nil { return creds, fmt.Errorf( "granting permissions to shared global bucket key: %w", err, ) } return creds, nil } func (n *network) getGarageBootstrapHosts( ctx context.Context, currBootstrap bootstrap.Bootstrap, ) ( map[nebula.HostName]bootstrap.Host, error, ) { garageClientParams, err := n.getGarageClientParams(ctx, currBootstrap) if err != nil { return nil, fmt.Errorf("getting garage client params: %w", err) } var ( client = garageClientParams.GlobalBucketS3APIClient() hosts = map[nebula.HostName]bootstrap.Host{} objInfoCh = client.ListObjects( ctx, garage.GlobalBucket, minio.ListObjectsOptions{ Prefix: garageGlobalBucketBootstrapHostsDirPath, Recursive: true, }, ) ) for objInfo := range objInfoCh { ctx := mctx.Annotate(ctx, "objectKey", objInfo.Key) if objInfo.Err != nil { return nil, fmt.Errorf("listing objects: %w", objInfo.Err) } obj, err := client.GetObject( ctx, garage.GlobalBucket, objInfo.Key, minio.GetObjectOptions{}, ) if err != nil { return nil, fmt.Errorf("retrieving object %q: %w", objInfo.Key, err) } var authedHost bootstrap.AuthenticatedHost err = json.NewDecoder(obj).Decode(&authedHost) obj.Close() if err != nil { n.logger.Warn(ctx, "Object contains invalid json", err) continue } host, err := authedHost.Unwrap(currBootstrap.CAPublicCredentials) if err != nil { n.logger.Warn(ctx, "Host could not be authenticated", err) } hosts[host.Name] = host } return hosts, nil } // putGarageBoostrapHost places the .json.signed file for this host // into garage so that other hosts are able to see relevant configuration for // it. func (n *network) putGarageBoostrapHost( ctx context.Context, currBootstrap bootstrap.Bootstrap, ) error { garageClientParams, err := n.getGarageClientParams(ctx, currBootstrap) if err != nil { return fmt.Errorf("getting garage client params: %w", err) } var ( host = currBootstrap.ThisHost() client = garageClientParams.GlobalBucketS3APIClient() ) configured, err := nebula.Sign( host.HostConfigured, currBootstrap.PrivateCredentials.SigningPrivateKey, ) if err != nil { return fmt.Errorf("signing host configured data: %w", err) } hostB, err := json.Marshal(bootstrap.AuthenticatedHost{ Assigned: currBootstrap.SignedHostAssigned, Configured: configured, }) if err != nil { return fmt.Errorf("encoding host data: %w", err) } filePath := filepath.Join( garageGlobalBucketBootstrapHostsDirPath, string(host.Name)+".json.signed", ) _, err = client.PutObject( ctx, garage.GlobalBucket, filePath, bytes.NewReader(hostB), int64(len(hostB)), minio.PutObjectOptions{}, ) if err != nil { return fmt.Errorf("writing to %q in global bucket: %w", filePath, err) } return nil } func removeGarageBootstrapHost( ctx context.Context, client garage.S3APIClient, hostName nebula.HostName, ) error { filePath := filepath.Join( garageGlobalBucketBootstrapHostsDirPath, string(hostName)+".json.signed", ) return client.RemoveObject( ctx, garage.GlobalBucket, filePath, minio.RemoveObjectOptions{}, ) }