isle/go/daemon/children/garage.go

package children

import (
	"context"
	"errors"
	"fmt"
	"isle/bootstrap"
	"isle/daemon/daecommon"
	"isle/garage"
	"isle/garage/garagesrv"
	"net"
	"path/filepath"
	"strconv"

	"code.betamike.com/micropelago/pmux/pmuxlib"
	"dev.mediocregopher.com/mediocre-go-lib.git/mctx"
	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
)

func garageAdminClientLogger(logger *mlog.Logger) *mlog.Logger {
	return logger.WithNamespace("garageAdminClient")
}

func garageAllocAdminAddr(
	host bootstrap.Host, alloc daecommon.ConfigStorageAllocation,
) string {
	return net.JoinHostPort(host.IP().String(), strconv.Itoa(alloc.AdminPort))
}

func waitForGarage(
	ctx context.Context,
	logger *mlog.Logger,
	adminToken string,
	procs map[string]garageProc,
	newCluster bool,
) error {
	adminClientLogger := garageAdminClientLogger(logger)

	for _, proc := range procs {
		adminClient := garage.NewAdminClient(
			adminClientLogger, proc.adminAddr, adminToken,
		)

		ctx := mctx.Annotate(
			ctx,
			"garageAdminAddr", proc.adminAddr,
			"garageDataPath", proc.alloc.DataPath,
		)

		logger.Info(ctx, "Waiting for garage instance to be healthy")

		err := adminClient.Wait(ctx, !newCluster)
		adminClient.Close()

		if err != nil {
			return fmt.Errorf(
				"waiting for garage instance %q to start up: %w",
				proc.adminAddr,
				err,
			)
		}
	}

	return nil

}

func garageWriteChildConfig(
	ctx context.Context,
	logger *mlog.Logger,
	rpcSecret, runtimeDirPath, adminToken string,
	hostBootstrap bootstrap.Bootstrap,
	alloc daecommon.ConfigStorageAllocation,
) (
	string, error,
) {
	var (
		thisHost = hostBootstrap.ThisHost()
		id       = daecommon.BootstrapGarageHostForAlloc(thisHost, alloc).ID

		node = garage.LocalNode{
			RemoteNode: garage.RemoteNode{
				ID:        id,
				IP:        thisHost.IP().String(),
				RPCPort:   alloc.RPCPort,
				S3APIPort: alloc.S3APIPort,
			},
			AdminPort: alloc.AdminPort,
		}

		garageTomlPath = filepath.Join(
			runtimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
		)
	)

	dbEngine, err := garagesrv.GetDBEngine(alloc.MetaPath)
	if err != nil {
		return "", fmt.Errorf("getting alloc db engine: %w", err)
	}

	err = garagesrv.WriteGarageTomlFile(
		ctx,
		logger,
		garageTomlPath,
		garagesrv.GarageTomlData{
			MetaPath: alloc.MetaPath,
			DataPath: alloc.DataPath,

			RPCSecret:  rpcSecret,
			AdminToken: adminToken,
			DBEngine:   dbEngine,

			LocalNode:      node,
			BootstrapPeers: hostBootstrap.GarageNodes(),
		},
	)

	if err != nil {
		return "", fmt.Errorf(
			"creating garage.toml file at %q: %w", garageTomlPath, err,
		)
	}

	return garageTomlPath, nil
}

func garagePmuxProcName(alloc daecommon.ConfigStorageAllocation) string {
	return fmt.Sprintf("garage-%d", alloc.RPCPort)
}

func garagePmuxProc(
	ctx context.Context,
	logger *mlog.Logger,
	binDirPath string,
	procName string,
	childConfigPath string,
) *pmuxlib.Process {
	cfg := pmuxlib.ProcessConfig{
		Cmd:  filepath.Join(binDirPath, "garage"),
		Args: []string{"-c", childConfigPath, "server"},
	}

	cfg = withPmuxLoggers(ctx, logger, procName, cfg)

	return pmuxlib.NewProcess(cfg)
}

func garagePmuxProcs(
	ctx context.Context,
	logger *mlog.Logger,
	rpcSecret, runtimeDirPath, binDirPath string,
	networkConfig daecommon.NetworkConfig,
	adminToken string,
	hostBootstrap bootstrap.Bootstrap,
) (
	map[string]garageProc, error,
) {
	var (
		procs    = map[string]garageProc{}
		allocs   = networkConfig.Storage.Allocations
		thisHost = hostBootstrap.ThisHost()
	)

	if len(allocs) > 0 && rpcSecret == "" {
		return nil, errors.New("Storage allocations defined, but garage RPC secret is not available")
	}

	for _, alloc := range allocs {
		childConfigPath, err := garageWriteChildConfig(
			ctx,
			logger,
			rpcSecret, runtimeDirPath, adminToken,
			hostBootstrap,
			alloc,
		)
		if err != nil {
			return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
		}

		procName := garagePmuxProcName(alloc)
		procs[procName] = garageProc{
			Process: garagePmuxProc(
				ctx, logger, binDirPath, procName, childConfigPath,
			),
			alloc:     alloc,
			adminAddr: garageAllocAdminAddr(thisHost, alloc),
		}
	}

	return procs, nil
}