isle/go/daemon/children/garage.go

148 lines
3.4 KiB
Go

package children
import (
"context"
"fmt"
"isle/bootstrap"
"isle/daemon/daecommon"
"isle/garage"
"isle/garage/garagesrv"
"net"
"path/filepath"
"strconv"
"code.betamike.com/micropelago/pmux/pmuxlib"
"dev.mediocregopher.com/mediocre-go-lib.git/mctx"
"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
)
func garageAdminClientLogger(logger *mlog.Logger) *mlog.Logger {
return logger.WithNamespace("garageAdminClient")
}
func waitForGarage(
ctx context.Context,
logger *mlog.Logger,
networkConfig daecommon.NetworkConfig,
adminToken string,
hostBootstrap bootstrap.Bootstrap,
) error {
allocs := networkConfig.Storage.Allocations
// if this host doesn't have any allocations specified then fall back to
// waiting for nebula
if len(allocs) == 0 {
return nil
}
adminClientLogger := garageAdminClientLogger(logger)
for _, alloc := range allocs {
adminAddr := net.JoinHostPort(
hostBootstrap.ThisHost().IP().String(),
strconv.Itoa(alloc.AdminPort),
)
adminClient := garage.NewAdminClient(
adminClientLogger, adminAddr, adminToken,
)
ctx := mctx.Annotate(ctx, "garageAdminAddr", adminAddr)
logger.Debug(ctx, "waiting for garage instance to start")
if err := adminClient.Wait(ctx); err != nil {
return fmt.Errorf("waiting for garage instance %q to start up: %w", adminAddr, err)
}
}
return nil
}
func garageWriteChildConfig(
rpcSecret, runtimeDirPath, adminToken string,
hostBootstrap bootstrap.Bootstrap,
alloc daecommon.ConfigStorageAllocation,
) (
string, error,
) {
thisHost := hostBootstrap.ThisHost()
id := daecommon.BootstrapGarageHostForAlloc(thisHost, alloc).ID
peer := garage.LocalPeer{
RemotePeer: garage.RemotePeer{
ID: id,
IP: thisHost.IP().String(),
RPCPort: alloc.RPCPort,
S3APIPort: alloc.S3APIPort,
},
AdminPort: alloc.AdminPort,
}
garageTomlPath := filepath.Join(
runtimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
)
err := garagesrv.WriteGarageTomlFile(garageTomlPath, garagesrv.GarageTomlData{
MetaPath: alloc.MetaPath,
DataPath: alloc.DataPath,
RPCSecret: rpcSecret,
AdminToken: adminToken,
LocalPeer: peer,
BootstrapPeers: hostBootstrap.GaragePeers(),
})
if err != nil {
return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
}
return garageTomlPath, nil
}
func garagePmuxProcConfigs(
ctx context.Context,
logger *mlog.Logger,
rpcSecret, runtimeDirPath, binDirPath string,
networkConfig daecommon.NetworkConfig,
adminToken string,
hostBootstrap bootstrap.Bootstrap,
) (
map[string]pmuxlib.ProcessConfig, error,
) {
var (
pmuxProcConfigs = map[string]pmuxlib.ProcessConfig{}
allocs = networkConfig.Storage.Allocations
)
if len(allocs) > 0 && rpcSecret == "" {
logger.WarnString(ctx, "Not starting garage instances for storage allocations, missing garage RPC secret")
return nil, nil
}
for _, alloc := range allocs {
childConfigPath, err := garageWriteChildConfig(
rpcSecret, runtimeDirPath, adminToken, hostBootstrap, alloc,
)
if err != nil {
return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
}
procName := fmt.Sprintf("garage-%d", alloc.RPCPort)
pmuxProcConfigs[procName] = pmuxlib.ProcessConfig{
Cmd: filepath.Join(binDirPath, "garage"),
Args: []string{"-c", childConfigPath, "server"},
StartAfterFunc: func(ctx context.Context) error {
return waitForNebula(ctx, logger, hostBootstrap)
},
}
}
return pmuxProcConfigs, nil
}