2024-09-07 13:46:59 +00:00
|
|
|
package children
|
2024-06-17 18:51:02 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2024-10-27 13:31:10 +00:00
|
|
|
"errors"
|
2024-06-17 18:51:02 +00:00
|
|
|
"fmt"
|
|
|
|
"isle/bootstrap"
|
2024-09-07 13:11:04 +00:00
|
|
|
"isle/daemon/daecommon"
|
2024-06-17 18:51:02 +00:00
|
|
|
"isle/garage"
|
|
|
|
"isle/garage/garagesrv"
|
|
|
|
"net"
|
|
|
|
"path/filepath"
|
|
|
|
"strconv"
|
|
|
|
|
|
|
|
"code.betamike.com/micropelago/pmux/pmuxlib"
|
2024-06-22 15:49:56 +00:00
|
|
|
"dev.mediocregopher.com/mediocre-go-lib.git/mctx"
|
|
|
|
"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
|
2024-06-17 18:51:02 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
func garageAdminClientLogger(logger *mlog.Logger) *mlog.Logger {
|
|
|
|
return logger.WithNamespace("garageAdminClient")
|
|
|
|
}
|
|
|
|
|
2024-07-06 13:36:48 +00:00
|
|
|
func waitForGarage(
|
|
|
|
ctx context.Context,
|
|
|
|
logger *mlog.Logger,
|
2024-09-10 20:51:33 +00:00
|
|
|
networkConfig daecommon.NetworkConfig,
|
2024-07-14 10:19:39 +00:00
|
|
|
adminToken string,
|
2024-07-06 13:36:48 +00:00
|
|
|
hostBootstrap bootstrap.Bootstrap,
|
|
|
|
) error {
|
2024-06-17 18:51:02 +00:00
|
|
|
|
2024-09-10 20:51:33 +00:00
|
|
|
allocs := networkConfig.Storage.Allocations
|
2024-06-17 18:51:02 +00:00
|
|
|
if len(allocs) == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2024-07-06 13:36:48 +00:00
|
|
|
adminClientLogger := garageAdminClientLogger(logger)
|
2024-06-17 18:51:02 +00:00
|
|
|
|
|
|
|
for _, alloc := range allocs {
|
|
|
|
|
|
|
|
adminAddr := net.JoinHostPort(
|
2024-07-06 13:36:48 +00:00
|
|
|
hostBootstrap.ThisHost().IP().String(),
|
2024-06-17 18:51:02 +00:00
|
|
|
strconv.Itoa(alloc.AdminPort),
|
|
|
|
)
|
|
|
|
|
|
|
|
adminClient := garage.NewAdminClient(
|
2024-07-14 10:19:39 +00:00
|
|
|
adminClientLogger, adminAddr, adminToken,
|
2024-06-17 18:51:02 +00:00
|
|
|
)
|
|
|
|
|
2024-10-27 13:31:10 +00:00
|
|
|
ctx := mctx.Annotate(
|
|
|
|
ctx, "garageAdminAddr", adminAddr, "garageDataPath", alloc.DataPath,
|
|
|
|
)
|
|
|
|
logger.Info(ctx, "Waiting for garage instance to be healthy")
|
2024-06-17 18:51:02 +00:00
|
|
|
|
|
|
|
if err := adminClient.Wait(ctx); err != nil {
|
|
|
|
return fmt.Errorf("waiting for garage instance %q to start up: %w", adminAddr, err)
|
|
|
|
}
|
2024-10-07 19:12:47 +00:00
|
|
|
|
|
|
|
adminClient.Close()
|
2024-06-17 18:51:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
func garageWriteChildConfig(
|
2024-10-27 13:31:10 +00:00
|
|
|
ctx context.Context,
|
|
|
|
logger *mlog.Logger,
|
2024-07-14 10:19:39 +00:00
|
|
|
rpcSecret, runtimeDirPath, adminToken string,
|
2024-06-17 18:51:02 +00:00
|
|
|
hostBootstrap bootstrap.Bootstrap,
|
2024-09-07 13:11:04 +00:00
|
|
|
alloc daecommon.ConfigStorageAllocation,
|
2024-06-17 18:51:02 +00:00
|
|
|
) (
|
2024-10-27 13:31:10 +00:00
|
|
|
string, bool, error,
|
2024-06-17 18:51:02 +00:00
|
|
|
) {
|
2024-10-27 13:31:10 +00:00
|
|
|
var (
|
|
|
|
thisHost = hostBootstrap.ThisHost()
|
|
|
|
id = daecommon.BootstrapGarageHostForAlloc(thisHost, alloc).ID
|
|
|
|
|
|
|
|
peer = garage.LocalPeer{
|
|
|
|
RemotePeer: garage.RemotePeer{
|
|
|
|
ID: id,
|
|
|
|
IP: thisHost.IP().String(),
|
|
|
|
RPCPort: alloc.RPCPort,
|
|
|
|
S3APIPort: alloc.S3APIPort,
|
|
|
|
},
|
|
|
|
AdminPort: alloc.AdminPort,
|
|
|
|
}
|
2024-06-17 18:51:02 +00:00
|
|
|
|
2024-10-27 13:31:10 +00:00
|
|
|
garageTomlPath = filepath.Join(
|
|
|
|
runtimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
|
|
|
|
)
|
2024-06-17 18:51:02 +00:00
|
|
|
)
|
|
|
|
|
2024-10-27 13:31:10 +00:00
|
|
|
changed, err := garagesrv.WriteGarageTomlFile(
|
|
|
|
ctx,
|
|
|
|
logger,
|
|
|
|
garageTomlPath,
|
|
|
|
garagesrv.GarageTomlData{
|
|
|
|
MetaPath: alloc.MetaPath,
|
|
|
|
DataPath: alloc.DataPath,
|
2024-06-17 18:51:02 +00:00
|
|
|
|
2024-10-27 13:31:10 +00:00
|
|
|
RPCSecret: rpcSecret,
|
|
|
|
AdminToken: adminToken,
|
2024-06-17 18:51:02 +00:00
|
|
|
|
2024-10-27 13:31:10 +00:00
|
|
|
LocalPeer: peer,
|
|
|
|
BootstrapPeers: hostBootstrap.GaragePeers(),
|
|
|
|
},
|
|
|
|
)
|
2024-06-17 18:51:02 +00:00
|
|
|
|
|
|
|
if err != nil {
|
2024-10-27 13:31:10 +00:00
|
|
|
return "", false, fmt.Errorf(
|
|
|
|
"creating garage.toml file at %q: %w", garageTomlPath, err,
|
|
|
|
)
|
2024-06-17 18:51:02 +00:00
|
|
|
}
|
|
|
|
|
2024-10-27 13:31:10 +00:00
|
|
|
return garageTomlPath, changed, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func garagePmuxProcName(alloc daecommon.ConfigStorageAllocation) string {
|
|
|
|
return fmt.Sprintf("garage-%d", alloc.RPCPort)
|
2024-06-17 18:51:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func garagePmuxProcConfigs(
|
2024-07-13 12:34:06 +00:00
|
|
|
ctx context.Context,
|
2024-07-06 13:36:48 +00:00
|
|
|
logger *mlog.Logger,
|
2024-07-13 12:34:06 +00:00
|
|
|
rpcSecret, runtimeDirPath, binDirPath string,
|
2024-09-10 20:51:33 +00:00
|
|
|
networkConfig daecommon.NetworkConfig,
|
2024-07-14 10:19:39 +00:00
|
|
|
adminToken string,
|
2024-07-06 13:36:48 +00:00
|
|
|
hostBootstrap bootstrap.Bootstrap,
|
2024-06-17 18:51:02 +00:00
|
|
|
) (
|
2024-07-19 14:50:20 +00:00
|
|
|
map[string]pmuxlib.ProcessConfig, error,
|
2024-06-17 18:51:02 +00:00
|
|
|
) {
|
2024-07-13 12:34:06 +00:00
|
|
|
var (
|
2024-07-19 14:50:20 +00:00
|
|
|
pmuxProcConfigs = map[string]pmuxlib.ProcessConfig{}
|
2024-09-10 20:51:33 +00:00
|
|
|
allocs = networkConfig.Storage.Allocations
|
2024-07-13 12:34:06 +00:00
|
|
|
)
|
2024-06-17 18:51:02 +00:00
|
|
|
|
2024-07-13 12:34:06 +00:00
|
|
|
if len(allocs) > 0 && rpcSecret == "" {
|
2024-10-27 13:31:10 +00:00
|
|
|
return nil, errors.New("Storage allocations defined, but garage RPC secret is not available")
|
2024-07-13 12:34:06 +00:00
|
|
|
}
|
2024-06-17 18:51:02 +00:00
|
|
|
|
2024-07-13 12:34:06 +00:00
|
|
|
for _, alloc := range allocs {
|
2024-10-27 13:31:10 +00:00
|
|
|
childConfigPath, _, err := garageWriteChildConfig(
|
|
|
|
ctx,
|
|
|
|
logger,
|
|
|
|
rpcSecret, runtimeDirPath, adminToken,
|
|
|
|
hostBootstrap,
|
|
|
|
alloc,
|
2024-06-17 18:51:02 +00:00
|
|
|
)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
|
|
|
|
}
|
|
|
|
|
2024-10-27 13:31:10 +00:00
|
|
|
procName := garagePmuxProcName(alloc)
|
2024-07-19 14:50:20 +00:00
|
|
|
pmuxProcConfigs[procName] = pmuxlib.ProcessConfig{
|
2024-06-17 18:51:02 +00:00
|
|
|
Cmd: filepath.Join(binDirPath, "garage"),
|
|
|
|
Args: []string{"-c", childConfigPath, "server"},
|
|
|
|
StartAfterFunc: func(ctx context.Context) error {
|
2024-07-06 13:36:48 +00:00
|
|
|
return waitForNebula(ctx, logger, hostBootstrap)
|
2024-06-17 18:51:02 +00:00
|
|
|
},
|
2024-07-19 14:50:20 +00:00
|
|
|
}
|
2024-06-17 18:51:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return pmuxProcConfigs, nil
|
|
|
|
}
|