190 lines
4.6 KiB
Go
190 lines
4.6 KiB
Go
|
package entrypoint
|
||
|
|
||
|
import (
|
||
|
"context"
|
||
|
crypticnet "cryptic-net"
|
||
|
"cryptic-net/garage"
|
||
|
"fmt"
|
||
|
"net"
|
||
|
"os"
|
||
|
"path/filepath"
|
||
|
"strconv"
|
||
|
"time"
|
||
|
|
||
|
"github.com/cryptic-io/pmux/pmuxlib"
|
||
|
)
|
||
|
|
||
|
func waitForGarageArgs(env *crypticnet.Env, args ...string) []string {
|
||
|
|
||
|
thisHost := env.Bootstrap.ThisHost()
|
||
|
allocs := env.ThisDaemon().Storage.Allocations
|
||
|
|
||
|
if len(allocs) == 0 {
|
||
|
return waitForNebulaArgs(env, args...)
|
||
|
}
|
||
|
|
||
|
var preArgs []string
|
||
|
|
||
|
for _, alloc := range allocs {
|
||
|
preArgs = append(
|
||
|
preArgs,
|
||
|
"wait-for",
|
||
|
net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
|
||
|
"--",
|
||
|
)
|
||
|
}
|
||
|
|
||
|
return append(preArgs, args...)
|
||
|
}
|
||
|
|
||
|
func garageWriteChildConf(
|
||
|
env *crypticnet.Env,
|
||
|
alloc crypticnet.DaemonYmlStorageAllocation,
|
||
|
) (
|
||
|
string, error,
|
||
|
) {
|
||
|
|
||
|
if err := os.MkdirAll(alloc.MetaPath, 0750); err != nil {
|
||
|
return "", fmt.Errorf("making directory %q: %w", alloc.MetaPath, err)
|
||
|
}
|
||
|
|
||
|
thisHost := env.Bootstrap.ThisHost()
|
||
|
|
||
|
peer := garage.Peer{
|
||
|
IP: thisHost.Nebula.IP,
|
||
|
RPCPort: alloc.RPCPort,
|
||
|
S3APIPort: alloc.S3APIPort,
|
||
|
}
|
||
|
|
||
|
pubKey, privKey := peer.RPCPeerKey()
|
||
|
|
||
|
nodeKeyPath := filepath.Join(alloc.MetaPath, "node_key")
|
||
|
nodeKeyPubPath := filepath.Join(alloc.MetaPath, "node_keypub")
|
||
|
|
||
|
if err := os.WriteFile(nodeKeyPath, privKey, 0400); err != nil {
|
||
|
return "", fmt.Errorf("writing private key to %q: %w", nodeKeyPath, err)
|
||
|
|
||
|
} else if err := os.WriteFile(nodeKeyPubPath, pubKey, 0440); err != nil {
|
||
|
return "", fmt.Errorf("writing public key to %q: %w", nodeKeyPubPath, err)
|
||
|
}
|
||
|
|
||
|
garageTomlPath := filepath.Join(
|
||
|
env.RuntimeDirPath, fmt.Sprintf("garage-%d.toml", alloc.RPCPort),
|
||
|
)
|
||
|
|
||
|
err := garage.WriteGarageTomlFile(garageTomlPath, garage.GarageTomlData{
|
||
|
MetaPath: alloc.MetaPath,
|
||
|
DataPath: alloc.DataPath,
|
||
|
|
||
|
RPCSecret: env.Bootstrap.GarageRPCSecret,
|
||
|
AdminToken: env.Bootstrap.GarageAdminToken,
|
||
|
|
||
|
RPCAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.RPCPort)),
|
||
|
APIAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.S3APIPort)),
|
||
|
AdminAddr: net.JoinHostPort(thisHost.Nebula.IP, strconv.Itoa(alloc.AdminPort)),
|
||
|
|
||
|
BootstrapPeers: env.Bootstrap.GarageRPCPeerAddrs(),
|
||
|
})
|
||
|
|
||
|
if err != nil {
|
||
|
return "", fmt.Errorf("creating garage.toml file at %q: %w", garageTomlPath, err)
|
||
|
}
|
||
|
|
||
|
return garageTomlPath, nil
|
||
|
}
|
||
|
|
||
|
func garageChildrenPmuxProcConfigs(env *crypticnet.Env) ([]pmuxlib.ProcessConfig, error) {
|
||
|
|
||
|
var pmuxProcConfigs []pmuxlib.ProcessConfig
|
||
|
|
||
|
for _, alloc := range env.ThisDaemon().Storage.Allocations {
|
||
|
|
||
|
childConfPath, err := garageWriteChildConf(env, alloc)
|
||
|
|
||
|
if err != nil {
|
||
|
return nil, fmt.Errorf("writing child config file for alloc %+v: %w", alloc, err)
|
||
|
}
|
||
|
|
||
|
pmuxProcConfigs = append(pmuxProcConfigs, pmuxlib.ProcessConfig{
|
||
|
Name: fmt.Sprintf("garage-%d", alloc.RPCPort),
|
||
|
Cmd: "garage",
|
||
|
Args: []string{"-c", childConfPath, "server"},
|
||
|
SigKillWait: 1 * time.Minute,
|
||
|
})
|
||
|
}
|
||
|
|
||
|
return pmuxProcConfigs, nil
|
||
|
}
|
||
|
|
||
|
func garageApplyLayoutDiffPmuxProcConfig(env *crypticnet.Env) pmuxlib.ProcessConfig {
|
||
|
return pmuxlib.ProcessConfig{
|
||
|
Name: "garage-apply-layout-diff",
|
||
|
Cmd: "bash",
|
||
|
Args: waitForGarageArgs(env, "bash", "garage-apply-layout-diff"),
|
||
|
NoRestartOn: []int{0},
|
||
|
}
|
||
|
}
|
||
|
|
||
|
func garageApplyLayout(
|
||
|
ctx context.Context,
|
||
|
adminClient *garage.AdminClient,
|
||
|
hostName, ipStr string,
|
||
|
allocs []crypticnet.DaemonYmlStorageAllocation,
|
||
|
) error {
|
||
|
|
||
|
type peerLayout struct {
|
||
|
Capacity int `json:"capacity"`
|
||
|
Zone string `json:"zone"`
|
||
|
Tags []string `json:"tags"`
|
||
|
}
|
||
|
|
||
|
{
|
||
|
clusterLayout := map[string]peerLayout{}
|
||
|
|
||
|
for _, alloc := range allocs {
|
||
|
|
||
|
peer := garage.Peer{
|
||
|
IP: ipStr,
|
||
|
RPCPort: alloc.RPCPort,
|
||
|
S3APIPort: alloc.S3APIPort,
|
||
|
}
|
||
|
|
||
|
clusterLayout[peer.RPCPeerID()] = peerLayout{
|
||
|
Capacity: alloc.Capacity / 100,
|
||
|
Zone: hostName,
|
||
|
}
|
||
|
}
|
||
|
|
||
|
err := adminClient.Do(ctx, nil, "POST", "/v0/layout", clusterLayout)
|
||
|
if err != nil {
|
||
|
return fmt.Errorf("staging layout changes: %w", err)
|
||
|
}
|
||
|
}
|
||
|
|
||
|
var clusterLayout struct {
|
||
|
Version int `json:"version"`
|
||
|
StagedRoleChanges map[string]peerLayout `json:"stagedRoleChanges"`
|
||
|
}
|
||
|
|
||
|
if err := adminClient.Do(ctx, &clusterLayout, "GET", "/v0/layout", nil); err != nil {
|
||
|
return fmt.Errorf("retrieving staged layout change: %w", err)
|
||
|
}
|
||
|
|
||
|
if len(clusterLayout.StagedRoleChanges) == 0 {
|
||
|
return nil
|
||
|
}
|
||
|
|
||
|
applyClusterLayout := struct {
|
||
|
Version int `json:"version"`
|
||
|
}{
|
||
|
Version: clusterLayout.Version + 1,
|
||
|
}
|
||
|
|
||
|
err := adminClient.Do(ctx, nil, "POST", "/v0/layout/apply", applyClusterLayout)
|
||
|
if err != nil {
|
||
|
return fmt.Errorf("applying new layout (new version:%d): %w", applyClusterLayout.Version, err)
|
||
|
}
|
||
|
|
||
|
return nil
|
||
|
}
|