148 lines
3.4 KiB
Go
148 lines
3.4 KiB
Go
package children
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"isle/bootstrap"
|
|
"isle/daemon/daecommon"
|
|
"isle/garage"
|
|
"isle/garage/garagesrv"
|
|
"net"
|
|
"path/filepath"
|
|
"strconv"
|
|
|
|
"code.betamike.com/micropelago/pmux/pmuxlib"
|
|
"dev.mediocregopher.com/mediocre-go-lib.git/mctx"
|
|
"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
|
|
)
|
|
|
|
func garageAdminClientLogger(logger *mlog.Logger) *mlog.Logger {
|
|
return logger.WithNamespace("garageAdminClient")
|
|
}
|
|
|
|
func waitForGarage(
|
|
ctx context.Context,
|
|
logger *mlog.Logger,
|
|
networkConfig daecommon.NetworkConfig,
|
|
adminToken string,
|
|
hostBootstrap bootstrap.Bootstrap,
|
|
) error {
|
|
|
|
allocs := networkConfig.Storage.Allocations
|
|
|
|
// if this host doesn't have any allocations specified then fall back to
|
|
// waiting for nebula
|
|
if len(allocs) == 0 {
|
|
return nil
|
|
}
|
|
|
|
adminClientLogger := garageAdminClientLogger(logger)
|
|
|
|
for _, alloc := range allocs {
|
|
|
|
adminAddr := net.JoinHostPort(
|
|
hostBootstrap.ThisHost().IP().String(),
|
|
strconv.Itoa(alloc.AdminPort),
|
|
)
|
|
|
|
adminClient := garage.NewAdminClient(
|
|
adminClientLogger, adminAddr, adminToken,
|
|
)
|
|
|
|
ctx := mctx.Annotate(ctx, "garageAdminAddr", adminAddr)
|
|
logger.Debug(ctx, "waiting for garage instance to start")
|
|
|
|
if err := adminClient.Wait(ctx); err != nil {
|
|
return fmt.Errorf("waiting for garage instance %q to start up: %w", adminAddr, err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
func garageWriteChildConfig(
|
|
rpcSecret, runtimeDirPath, adminToken string,
|
|
hostBootstrap bootstrap.Bootstrap,
|
|
alloc daecommon.ConfigStorageAllocation,
|
|
) (
|
|
string, error,
|
|
) {
|
|
|
|
thisHost := hostBootstrap.ThisHost()
|
|
id := daecommon.BootstrapGarageHostForAlloc(thisHost, alloc).ID
|
|
|
|
peer := garage.LocalPeer{
|
|
RemotePeer: garage.RemotePeer{
|
|
ID: id,
|
|
IP: thisHost.IP().String(),
|
|
RPCPort: alloc.RPCPort,
|
|
S3APIPort: alloc.S3APIPort,
|
|
},
|
|
AdminPort: alloc.AdminPort,
|
|
}
|
|
|
|
garageTomlPath := filepath.Join(
|
|
runtimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
|
|
)
|
|
|
|
err := garagesrv.WriteGarageTomlFile(garageTomlPath, garagesrv.GarageTomlData{
|
|
MetaPath: alloc.MetaPath,
|
|
DataPath: alloc.DataPath,
|
|
|
|
RPCSecret: rpcSecret,
|
|
AdminToken: adminToken,
|
|
|
|
LocalPeer: peer,
|
|
BootstrapPeers: hostBootstrap.GaragePeers(),
|
|
})
|
|
|
|
if err != nil {
|
|
return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
|
|
}
|
|
|
|
return garageTomlPath, nil
|
|
}
|
|
|
|
func garagePmuxProcConfigs(
|
|
ctx context.Context,
|
|
logger *mlog.Logger,
|
|
rpcSecret, runtimeDirPath, binDirPath string,
|
|
networkConfig daecommon.NetworkConfig,
|
|
adminToken string,
|
|
hostBootstrap bootstrap.Bootstrap,
|
|
) (
|
|
map[string]pmuxlib.ProcessConfig, error,
|
|
) {
|
|
var (
|
|
pmuxProcConfigs = map[string]pmuxlib.ProcessConfig{}
|
|
allocs = networkConfig.Storage.Allocations
|
|
)
|
|
|
|
if len(allocs) > 0 && rpcSecret == "" {
|
|
logger.WarnString(ctx, "Not starting garage instances for storage allocations, missing garage RPC secret")
|
|
return nil, nil
|
|
}
|
|
|
|
for _, alloc := range allocs {
|
|
|
|
childConfigPath, err := garageWriteChildConfig(
|
|
rpcSecret, runtimeDirPath, adminToken, hostBootstrap, alloc,
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
|
|
}
|
|
|
|
procName := fmt.Sprintf("garage-%d", alloc.RPCPort)
|
|
pmuxProcConfigs[procName] = pmuxlib.ProcessConfig{
|
|
Cmd: filepath.Join(binDirPath, "garage"),
|
|
Args: []string{"-c", childConfigPath, "server"},
|
|
StartAfterFunc: func(ctx context.Context) error {
|
|
return waitForNebula(ctx, logger, hostBootstrap)
|
|
},
|
|
}
|
|
}
|
|
|
|
return pmuxProcConfigs, nil
|
|
}
|