isle/go/daemon/children/garage.go

package children

import (
	"context"
	"errors"
	"fmt"
	"isle/bootstrap"
	"isle/daemon/daecommon"
	"isle/garage"
	"isle/garage/garagesrv"
	"net"
	"path/filepath"
	"strconv"

	"code.betamike.com/micropelago/pmux/pmuxlib"
	"dev.mediocregopher.com/mediocre-go-lib.git/mctx"
	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
)

func garageAdminClientLogger(logger *mlog.Logger) *mlog.Logger {
	return logger.WithNamespace("garageAdminClient")
}

func waitForGarage(
	ctx context.Context,
	logger *mlog.Logger,
	networkConfig daecommon.NetworkConfig,
	adminToken string,
	hostBootstrap bootstrap.Bootstrap,
) error {

	allocs := networkConfig.Storage.Allocations
	if len(allocs) == 0 {
		return nil
	}

	adminClientLogger := garageAdminClientLogger(logger)

	for _, alloc := range allocs {

		adminAddr := net.JoinHostPort(
			hostBootstrap.ThisHost().IP().String(),
			strconv.Itoa(alloc.AdminPort),
		)

		adminClient := garage.NewAdminClient(
			adminClientLogger, adminAddr, adminToken,
		)

		ctx := mctx.Annotate(
			ctx, "garageAdminAddr", adminAddr, "garageDataPath", alloc.DataPath,
		)
		logger.Info(ctx, "Waiting for garage instance to be healthy")

		if err := adminClient.Wait(ctx); err != nil {
			return fmt.Errorf("waiting for garage instance %q to start up: %w", adminAddr, err)
		}

		adminClient.Close()
	}

	return nil

}

func garageWriteChildConfig(
	ctx context.Context,
	logger *mlog.Logger,
	rpcSecret, runtimeDirPath, adminToken string,
	hostBootstrap bootstrap.Bootstrap,
	alloc daecommon.ConfigStorageAllocation,
) (
	string, bool, error,
) {
	var (
		thisHost = hostBootstrap.ThisHost()
		id       = daecommon.BootstrapGarageHostForAlloc(thisHost, alloc).ID

		peer = garage.LocalPeer{
			RemotePeer: garage.RemotePeer{
				ID:        id,
				IP:        thisHost.IP().String(),
				RPCPort:   alloc.RPCPort,
				S3APIPort: alloc.S3APIPort,
			},
			AdminPort: alloc.AdminPort,
		}

		garageTomlPath = filepath.Join(
			runtimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
		)
	)

	changed, err := garagesrv.WriteGarageTomlFile(
		ctx,
		logger,
		garageTomlPath,
		garagesrv.GarageTomlData{
			MetaPath: alloc.MetaPath,
			DataPath: alloc.DataPath,

			RPCSecret:  rpcSecret,
			AdminToken: adminToken,

			LocalPeer:      peer,
			BootstrapPeers: hostBootstrap.GaragePeers(),
		},
	)

	if err != nil {
		return "", false, fmt.Errorf(
			"creating garage.toml file at %q: %w", garageTomlPath, err,
		)
	}

	return garageTomlPath, changed, nil
}

func garagePmuxProcName(alloc daecommon.ConfigStorageAllocation) string {
	return fmt.Sprintf("garage-%d", alloc.RPCPort)
}

func garagePmuxProcConfigs(
	ctx context.Context,
	logger *mlog.Logger,
	rpcSecret, runtimeDirPath, binDirPath string,
	networkConfig daecommon.NetworkConfig,
	adminToken string,
	hostBootstrap bootstrap.Bootstrap,
) (
	map[string]pmuxlib.ProcessConfig, error,
) {
	var (
		pmuxProcConfigs = map[string]pmuxlib.ProcessConfig{}
		allocs          = networkConfig.Storage.Allocations
	)

	if len(allocs) > 0 && rpcSecret == "" {
		return nil, errors.New("Storage allocations defined, but garage RPC secret is not available")
	}

	for _, alloc := range allocs {
		childConfigPath, _, err := garageWriteChildConfig(
			ctx,
			logger,
			rpcSecret, runtimeDirPath, adminToken,
			hostBootstrap,
			alloc,
		)
		if err != nil {
			return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
		}

		procName := garagePmuxProcName(alloc)
		pmuxProcConfigs[procName] = pmuxlib.ProcessConfig{
			Cmd:  filepath.Join(binDirPath, "garage"),
			Args: []string{"-c", childConfigPath, "server"},
			StartAfterFunc: func(ctx context.Context) error {
				return waitForNebula(ctx, logger, hostBootstrap)
			},
		}
	}

	return pmuxProcConfigs, nil
}