isle/go/daemon/child_garage.go

package daemon

import (
	"context"
	"fmt"
	"isle/bootstrap"
	"isle/garage"
	"isle/garage/garagesrv"
	"net"
	"path/filepath"
	"strconv"

	"code.betamike.com/micropelago/pmux/pmuxlib"
	"dev.mediocregopher.com/mediocre-go-lib.git/mctx"
	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
)

func garageAdminClientLogger(logger *mlog.Logger) *mlog.Logger {
	return logger.WithNamespace("garageAdminClient")
}

// newGarageAdminClient will return an AdminClient for a local garage instance,
// or it will _panic_ if there is no local instance configured.
func newGarageAdminClient(
	logger *mlog.Logger,
	daemonConfig Config,
	adminToken string,
	hostBootstrap bootstrap.Bootstrap,
) *garage.AdminClient {

	thisHost := hostBootstrap.ThisHost()

	return garage.NewAdminClient(
		garageAdminClientLogger(logger),
		net.JoinHostPort(
			thisHost.IP().String(),
			strconv.Itoa(daemonConfig.Storage.Allocations[0].AdminPort),
		),
		adminToken,
	)
}

func waitForGarage(
	ctx context.Context,
	logger *mlog.Logger,
	daemonConfig Config,
	adminToken string,
	hostBootstrap bootstrap.Bootstrap,
) error {

	allocs := daemonConfig.Storage.Allocations

	// if this host doesn't have any allocations specified then fall back to
	// waiting for nebula
	if len(allocs) == 0 {
		return nil
	}

	adminClientLogger := garageAdminClientLogger(logger)

	for _, alloc := range allocs {

		adminAddr := net.JoinHostPort(
			hostBootstrap.ThisHost().IP().String(),
			strconv.Itoa(alloc.AdminPort),
		)

		adminClient := garage.NewAdminClient(
			adminClientLogger, adminAddr, adminToken,
		)

		ctx := mctx.Annotate(ctx, "garageAdminAddr", adminAddr)
		logger.Debug(ctx, "waiting for garage instance to start")

		if err := adminClient.Wait(ctx); err != nil {
			return fmt.Errorf("waiting for garage instance %q to start up: %w", adminAddr, err)
		}
	}

	return nil

}

// bootstrapGarageHostForAlloc returns the bootstrap.GarageHostInstance which
// corresponds with the given alloc from the daemon config. This will panic if
// no associated instance can be found.
//
// This assumes that coalesceDaemonConfigAndBootstrap has already been called.
func bootstrapGarageHostForAlloc(
	host bootstrap.Host,
	alloc ConfigStorageAllocation,
) bootstrap.GarageHostInstance {

	for _, inst := range host.Garage.Instances {
		if inst.RPCPort == alloc.RPCPort {
			return inst
		}
	}

	panic(fmt.Sprintf("could not find alloc %+v in the bootstrap data", alloc))
}

func garageWriteChildConfig(
	rpcSecret, runtimeDirPath, adminToken string,
	hostBootstrap bootstrap.Bootstrap,
	alloc ConfigStorageAllocation,
) (
	string, error,
) {

	thisHost := hostBootstrap.ThisHost()
	id := bootstrapGarageHostForAlloc(thisHost, alloc).ID

	peer := garage.LocalPeer{
		RemotePeer: garage.RemotePeer{
			ID:        id,
			IP:        thisHost.IP().String(),
			RPCPort:   alloc.RPCPort,
			S3APIPort: alloc.S3APIPort,
		},
		AdminPort: alloc.AdminPort,
	}

	garageTomlPath := filepath.Join(
		runtimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
	)

	err := garagesrv.WriteGarageTomlFile(garageTomlPath, garagesrv.GarageTomlData{
		MetaPath: alloc.MetaPath,
		DataPath: alloc.DataPath,

		RPCSecret:  rpcSecret,
		AdminToken: adminToken,

		LocalPeer:      peer,
		BootstrapPeers: hostBootstrap.GaragePeers(),
	})

	if err != nil {
		return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
	}

	return garageTomlPath, nil
}

func garagePmuxProcConfigs(
	ctx context.Context,
	logger *mlog.Logger,
	rpcSecret, runtimeDirPath, binDirPath string,
	daemonConfig Config,
	adminToken string,
	hostBootstrap bootstrap.Bootstrap,
) (
	map[string]pmuxlib.ProcessConfig, error,
) {
	var (
		pmuxProcConfigs = map[string]pmuxlib.ProcessConfig{}
		allocs          = daemonConfig.Storage.Allocations
	)

	if len(allocs) > 0 && rpcSecret == "" {
		logger.WarnString(ctx, "Not starting garage instances for storage allocations, missing garage RPC secret")
		return nil, nil
	}

	for _, alloc := range allocs {

		childConfigPath, err := garageWriteChildConfig(
			rpcSecret, runtimeDirPath, adminToken, hostBootstrap, alloc,
		)
		if err != nil {
			return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
		}

		procName := fmt.Sprintf("garage-%d", alloc.RPCPort)
		pmuxProcConfigs[procName] = pmuxlib.ProcessConfig{
			Cmd:  filepath.Join(binDirPath, "garage"),
			Args: []string{"-c", childConfigPath, "server"},
			StartAfterFunc: func(ctx context.Context) error {
				return waitForNebula(ctx, logger, hostBootstrap)
			},
		}
	}

	return pmuxProcConfigs, nil
}

func garageApplyLayout(
	ctx context.Context,
	logger *mlog.Logger,
	daemonConfig Config,
	adminToken string,
	hostBootstrap bootstrap.Bootstrap,
) error {

	var (
		adminClient = newGarageAdminClient(
			logger, daemonConfig, adminToken, hostBootstrap,
		)
		thisHost = hostBootstrap.ThisHost()
		hostName = thisHost.Name
		allocs   = daemonConfig.Storage.Allocations
		peers    = make([]garage.PeerLayout, len(allocs))
	)

	for i, alloc := range allocs {

		id := bootstrapGarageHostForAlloc(thisHost, alloc).ID

		zone := string(hostName)
		if alloc.Zone != "" {
			zone = alloc.Zone
		}

		peers[i] = garage.PeerLayout{
			ID:       id,
			Capacity: alloc.Capacity * 1_000_000_000,
			Zone:     zone,
			Tags:     []string{},
		}
	}

	return adminClient.ApplyLayout(ctx, peers)
}