Factor daemon.Children into its own package

This commit is contained in:
Brian Picciano 2024-09-07 15:46:59 +02:00
parent a840d0e701
commit 86b2ba7bfa
12 changed files with 269 additions and 216 deletions

View File

@ -1,13 +1,10 @@
package daemon package daemon
import ( import (
"bytes"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"reflect"
"isle/bootstrap" "isle/bootstrap"
"isle/daemon/daecommon" "isle/daemon/daecommon"
@ -81,49 +78,3 @@ func coalesceDaemonConfigAndBootstrap(
return hostBootstrap, nil return hostBootstrap, nil
} }
type bootstrapDiff struct {
hostsChanged bool
nebulaChanged bool
dnsChanged bool
}
func calcBootstrapDiff(
daemonConfig daecommon.Config,
prevBootstrap, nextBootstrap bootstrap.Bootstrap,
) (
diff bootstrapDiff, err error,
) {
{
prevHash, prevErr := bootstrap.HostsHash(prevBootstrap.Hosts)
nextHash, nextErr := bootstrap.HostsHash(nextBootstrap.Hosts)
if err = errors.Join(prevErr, nextErr); err != nil {
err = fmt.Errorf("calculating host hashes: %w", err)
return
}
diff.hostsChanged = !bytes.Equal(prevHash, nextHash)
}
{
prevNebulaConfig, prevErr := nebulaConfig(daemonConfig, prevBootstrap)
nextNebulaConfig, nextErr := nebulaConfig(daemonConfig, nextBootstrap)
if err = errors.Join(prevErr, nextErr); err != nil {
err = fmt.Errorf("calculating nebula config: %w", err)
return
}
diff.nebulaChanged = !reflect.DeepEqual(
prevNebulaConfig, nextNebulaConfig,
)
}
{
diff.dnsChanged = !reflect.DeepEqual(
dnsmasqConfig(daemonConfig, prevBootstrap),
dnsmasqConfig(daemonConfig, nextBootstrap),
)
}
return
}

View File

@ -1,17 +1,46 @@
package daemon // Package children manages the creation, lifetime, and shutdown of child
// processes created by the daemon.
package children
import ( import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"isle/bootstrap" "io"
"isle/daemon/daecommon" "os"
"isle/secrets"
"code.betamike.com/micropelago/pmux/pmuxlib" "code.betamike.com/micropelago/pmux/pmuxlib"
"dev.mediocregopher.com/mediocre-go-lib.git/mlog" "dev.mediocregopher.com/mediocre-go-lib.git/mlog"
"isle/bootstrap"
"isle/daemon/daecommon"
"isle/secrets"
) )
// Opts are optional parameters which can be passed in when initializing a new
// Children instance. A nil Opts is equivalent to a zero value.
type Opts struct {
// Stdout and Stderr are what the associated outputs from child processes
// will be directed to.
Stdout, Stderr io.Writer
}
func (o *Opts) withDefaults() *Opts {
if o == nil {
o = new(Opts)
}
if o.Stdout == nil {
o.Stdout = os.Stdout
}
if o.Stderr == nil {
o.Stderr = os.Stderr
}
return o
}
// Children manages all child processes of a network. Child processes are // Children manages all child processes of a network. Child processes are
// comprised of: // comprised of:
// - nebula // - nebula
@ -20,19 +49,21 @@ import (
type Children struct { type Children struct {
logger *mlog.Logger logger *mlog.Logger
daemonConfig daecommon.Config daemonConfig daecommon.Config
runtimeDirPath string
opts Opts opts Opts
pmux *pmuxlib.Pmux pmux *pmuxlib.Pmux
} }
// NewChildren initialized and returns a Children instance. If initialization // New initializes and returns a Children instance. If initialization fails an
// fails an error is returned. // error is returned.
func NewChildren( func New(
ctx context.Context, ctx context.Context,
logger *mlog.Logger, logger *mlog.Logger,
binDirPath string, binDirPath string,
secretsStore secrets.Store, secretsStore secrets.Store,
daemonConfig daecommon.Config, daemonConfig daecommon.Config,
runtimeDirPath string,
garageAdminToken string, garageAdminToken string,
hostBootstrap bootstrap.Bootstrap, hostBootstrap bootstrap.Bootstrap,
opts *Opts, opts *Opts,
@ -50,6 +81,7 @@ func NewChildren(
c := &Children{ c := &Children{
logger: logger, logger: logger,
daemonConfig: daemonConfig, daemonConfig: daemonConfig,
runtimeDirPath: runtimeDirPath,
opts: *opts, opts: *opts,
} }
@ -80,9 +112,12 @@ func NewChildren(
} }
// RestartDNSMasq rewrites the dnsmasq config and restarts the process. // RestartDNSMasq rewrites the dnsmasq config and restarts the process.
//
// TODO block until process has been confirmed to have come back up
// successfully.
func (c *Children) RestartDNSMasq(hostBootstrap bootstrap.Bootstrap) error { func (c *Children) RestartDNSMasq(hostBootstrap bootstrap.Bootstrap) error {
_, err := dnsmasqWriteConfig( _, err := dnsmasqWriteConfig(
c.opts.EnvVars.RuntimeDirPath, c.daemonConfig, hostBootstrap, c.runtimeDirPath, c.daemonConfig, hostBootstrap,
) )
if err != nil { if err != nil {
return fmt.Errorf("writing new dnsmasq config: %w", err) return fmt.Errorf("writing new dnsmasq config: %w", err)
@ -93,9 +128,12 @@ func (c *Children) RestartDNSMasq(hostBootstrap bootstrap.Bootstrap) error {
} }
// RestartNebula rewrites the nebula config and restarts the process. // RestartNebula rewrites the nebula config and restarts the process.
//
// TODO block until process has been confirmed to have come back up
// successfully.
func (c *Children) RestartNebula(hostBootstrap bootstrap.Bootstrap) error { func (c *Children) RestartNebula(hostBootstrap bootstrap.Bootstrap) error {
_, err := nebulaWriteConfig( _, err := nebulaWriteConfig(
c.opts.EnvVars.RuntimeDirPath, c.daemonConfig, hostBootstrap, c.runtimeDirPath, c.daemonConfig, hostBootstrap,
) )
if err != nil { if err != nil {
return fmt.Errorf("writing a new nebula config: %w", err) return fmt.Errorf("writing a new nebula config: %w", err)
@ -105,6 +143,30 @@ func (c *Children) RestartNebula(hostBootstrap bootstrap.Bootstrap) error {
return nil return nil
} }
// Reload applies a ReloadDiff to the Children, using the given bootstrap which
// must be the same one which was passed to CalculateReloadDiff.
func (c *Children) Reload(
ctx context.Context, newBootstrap bootstrap.Bootstrap, diff ReloadDiff,
) error {
var errs []error
if diff.DNSChanged {
c.logger.Info(ctx, "Restarting dnsmasq to account for bootstrap changes")
if err := c.RestartDNSMasq(newBootstrap); err != nil {
errs = append(errs, fmt.Errorf("restarting dnsmasq: %w", err))
}
}
if diff.NebulaChanged {
c.logger.Info(ctx, "Restarting nebula to account for bootstrap changes")
if err := c.RestartNebula(newBootstrap); err != nil {
errs = append(errs, fmt.Errorf("restarting nebula: %w", err))
}
}
return errors.Join(errs...)
}
// Shutdown blocks until all child processes have gracefully shut themselves // Shutdown blocks until all child processes have gracefully shut themselves
// down. // down.
func (c *Children) Shutdown() { func (c *Children) Shutdown() {

View File

@ -0,0 +1,47 @@
package children
import (
"errors"
"fmt"
"isle/bootstrap"
"isle/daemon/daecommon"
"reflect"
)
// ReloadDiff describes which children had their configurations changed as part
// of a change in the bootstrap.
type ReloadDiff struct {
NebulaChanged bool
DNSChanged bool
}
// CalculateReloadDiff calculates a ReloadDiff based on an old and new
// bootstrap.
func CalculateReloadDiff(
daemonConfig daecommon.Config,
prevBootstrap, nextBootstrap bootstrap.Bootstrap,
) (
diff ReloadDiff, err error,
) {
{
prevNebulaConfig, prevErr := nebulaConfig(daemonConfig, prevBootstrap)
nextNebulaConfig, nextErr := nebulaConfig(daemonConfig, nextBootstrap)
if err = errors.Join(prevErr, nextErr); err != nil {
err = fmt.Errorf("calculating nebula config: %w", err)
return
}
diff.NebulaChanged = !reflect.DeepEqual(
prevNebulaConfig, nextNebulaConfig,
)
}
{
diff.DNSChanged = !reflect.DeepEqual(
dnsmasqConfig(daemonConfig, prevBootstrap),
dnsmasqConfig(daemonConfig, nextBootstrap),
)
}
return
}

View File

@ -1,4 +1,4 @@
package daemon package children
import ( import (
"context" "context"

View File

@ -1,4 +1,4 @@
package daemon package children
import ( import (
"context" "context"
@ -20,27 +20,6 @@ func garageAdminClientLogger(logger *mlog.Logger) *mlog.Logger {
return logger.WithNamespace("garageAdminClient") return logger.WithNamespace("garageAdminClient")
} }
// newGarageAdminClient will return an AdminClient for a local garage instance,
// or it will _panic_ if there is no local instance configured.
func newGarageAdminClient(
logger *mlog.Logger,
daemonConfig daecommon.Config,
adminToken string,
hostBootstrap bootstrap.Bootstrap,
) *garage.AdminClient {
thisHost := hostBootstrap.ThisHost()
return garage.NewAdminClient(
garageAdminClientLogger(logger),
net.JoinHostPort(
thisHost.IP().String(),
strconv.Itoa(daemonConfig.Storage.Allocations[0].AdminPort),
),
adminToken,
)
}
func waitForGarage( func waitForGarage(
ctx context.Context, ctx context.Context,
logger *mlog.Logger, logger *mlog.Logger,
@ -82,25 +61,6 @@ func waitForGarage(
} }
// bootstrapGarageHostForAlloc returns the bootstrap.GarageHostInstance which
// corresponds with the given alloc from the daemon config. This will panic if
// no associated instance can be found.
//
// This assumes that coalesceDaemonConfigAndBootstrap has already been called.
func bootstrapGarageHostForAlloc(
host bootstrap.Host,
alloc daecommon.ConfigStorageAllocation,
) bootstrap.GarageHostInstance {
for _, inst := range host.Garage.Instances {
if inst.RPCPort == alloc.RPCPort {
return inst
}
}
panic(fmt.Sprintf("could not find alloc %+v in the bootstrap data", alloc))
}
func garageWriteChildConfig( func garageWriteChildConfig(
rpcSecret, runtimeDirPath, adminToken string, rpcSecret, runtimeDirPath, adminToken string,
hostBootstrap bootstrap.Bootstrap, hostBootstrap bootstrap.Bootstrap,
@ -110,7 +70,7 @@ func garageWriteChildConfig(
) { ) {
thisHost := hostBootstrap.ThisHost() thisHost := hostBootstrap.ThisHost()
id := bootstrapGarageHostForAlloc(thisHost, alloc).ID id := daecommon.BootstrapGarageHostForAlloc(thisHost, alloc).ID
peer := garage.LocalPeer{ peer := garage.LocalPeer{
RemotePeer: garage.RemotePeer{ RemotePeer: garage.RemotePeer{
@ -185,41 +145,3 @@ func garagePmuxProcConfigs(
return pmuxProcConfigs, nil return pmuxProcConfigs, nil
} }
func garageApplyLayout(
ctx context.Context,
logger *mlog.Logger,
daemonConfig daecommon.Config,
adminToken string,
hostBootstrap bootstrap.Bootstrap,
) error {
var (
adminClient = newGarageAdminClient(
logger, daemonConfig, adminToken, hostBootstrap,
)
thisHost = hostBootstrap.ThisHost()
hostName = thisHost.Name
allocs = daemonConfig.Storage.Allocations
peers = make([]garage.PeerLayout, len(allocs))
)
for i, alloc := range allocs {
id := bootstrapGarageHostForAlloc(thisHost, alloc).ID
zone := string(hostName)
if alloc.Zone != "" {
zone = alloc.Zone
}
peers[i] = garage.PeerLayout{
ID: id,
Capacity: alloc.Capacity * 1_000_000_000,
Zone: zone,
Tags: []string{},
}
}
return adminClient.ApplyLayout(ctx, peers)
}

View File

@ -0,0 +1,30 @@
package children
import (
"context"
"time"
"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
)
// until keeps trying fn until it returns nil, returning true. If the context is
// canceled then until returns false.
func until(
ctx context.Context,
logger *mlog.Logger,
descr string,
fn func(context.Context) error,
) bool {
for {
logger.Info(ctx, descr)
err := fn(ctx)
if err == nil {
return true
} else if ctxErr := ctx.Err(); ctxErr != nil {
return false
}
logger.Warn(ctx, descr+" failed, retrying in one second", err)
time.Sleep(1 * time.Second)
}
}

View File

@ -1,4 +1,4 @@
package daemon package children
import ( import (
"context" "context"

View File

@ -1,4 +1,4 @@
package daemon package children
import ( import (
"context" "context"
@ -19,7 +19,7 @@ func (c *Children) newPmuxConfig(
pmuxlib.Config, error, pmuxlib.Config, error,
) { ) {
nebulaPmuxProcConfig, err := nebulaPmuxProcConfig( nebulaPmuxProcConfig, err := nebulaPmuxProcConfig(
c.opts.EnvVars.RuntimeDirPath, c.runtimeDirPath,
binDirPath, binDirPath,
daemonConfig, daemonConfig,
hostBootstrap, hostBootstrap,
@ -30,7 +30,7 @@ func (c *Children) newPmuxConfig(
dnsmasqPmuxProcConfig, err := dnsmasqPmuxProcConfig( dnsmasqPmuxProcConfig, err := dnsmasqPmuxProcConfig(
c.logger, c.logger,
c.opts.EnvVars.RuntimeDirPath, c.runtimeDirPath,
binDirPath, binDirPath,
daemonConfig, daemonConfig,
hostBootstrap, hostBootstrap,
@ -45,7 +45,7 @@ func (c *Children) newPmuxConfig(
ctx, ctx,
c.logger, c.logger,
garageRPCSecret, garageRPCSecret,
c.opts.EnvVars.RuntimeDirPath, c.runtimeDirPath,
binDirPath, binDirPath,
daemonConfig, daemonConfig,
garageAdminToken, garageAdminToken,

View File

@ -3,6 +3,7 @@ package daecommon
import ( import (
"fmt" "fmt"
"io" "io"
"isle/bootstrap"
"isle/yamlutil" "isle/yamlutil"
"os" "os"
"path/filepath" "path/filepath"
@ -181,3 +182,20 @@ func LoadConfig(
return config, nil return config, nil
} }
// BootstrapGarageHostForAlloc returns the bootstrap.GarageHostInstance which
// corresponds with the given alloc from the daemon config. This will panic if
// no associated instance can be found.
func BootstrapGarageHostForAlloc(
host bootstrap.Host,
alloc ConfigStorageAllocation,
) bootstrap.GarageHostInstance {
for _, inst := range host.Garage.Instances {
if inst.RPCPort == alloc.RPCPort {
return inst
}
}
panic(fmt.Sprintf("could not find alloc %+v in the bootstrap data", alloc))
}

View File

@ -9,15 +9,14 @@ import (
"crypto/rand" "crypto/rand"
"errors" "errors"
"fmt" "fmt"
"io"
"io/fs" "io/fs"
"isle/bootstrap" "isle/bootstrap"
"isle/daemon/children"
"isle/daemon/daecommon" "isle/daemon/daecommon"
"isle/jsonutil" "isle/jsonutil"
"isle/nebula" "isle/nebula"
"isle/secrets" "isle/secrets"
"net/netip" "net/netip"
"os"
"path/filepath" "path/filepath"
"slices" "slices"
"sync" "sync"
@ -30,10 +29,7 @@ import (
// Opts are optional parameters which can be passed in when initializing a new // Opts are optional parameters which can be passed in when initializing a new
// Daemon instance. A nil Opts is equivalent to a zero value. // Daemon instance. A nil Opts is equivalent to a zero value.
type Opts struct { type Opts struct {
// Stdout and Stderr are what the associated outputs from child processes ChildrenOpts *children.Opts
// will be directed to.
Stdout, Stderr io.Writer
// Defaults to that returned by daecommon.GetEnvVars. // Defaults to that returned by daecommon.GetEnvVars.
EnvVars daecommon.EnvVars EnvVars daecommon.EnvVars
} }
@ -43,14 +39,6 @@ func (o *Opts) withDefaults() *Opts {
o = new(Opts) o = new(Opts)
} }
if o.Stdout == nil {
o.Stdout = os.Stdout
}
if o.Stderr == nil {
o.Stderr = os.Stderr
}
if o.EnvVars == (daecommon.EnvVars{}) { if o.EnvVars == (daecommon.EnvVars{}) {
o.EnvVars = daecommon.GetEnvVars() o.EnvVars = daecommon.GetEnvVars()
} }
@ -94,7 +82,7 @@ type Daemon struct {
l sync.RWMutex l sync.RWMutex
state int state int
children *Children children *children.Children
currBootstrap bootstrap.Bootstrap currBootstrap bootstrap.Bootstrap
shutdownCh chan struct{} shutdownCh chan struct{}
@ -198,15 +186,16 @@ func (d *Daemon) initialize(
d.state = daemonStateInitializing d.state = daemonStateInitializing
d.logger.Info(ctx, "Creating child processes") d.logger.Info(ctx, "Creating child processes")
d.children, err = NewChildren( d.children, err = children.New(
ctx, ctx,
d.logger.WithNamespace("children"), d.logger.WithNamespace("children"),
d.envBinDirPath, d.envBinDirPath,
d.secretsStore, d.secretsStore,
d.daemonConfig, d.daemonConfig,
d.opts.EnvVars.RuntimeDirPath,
d.garageAdminToken, d.garageAdminToken,
currBootstrap, currBootstrap,
d.opts, d.opts.ChildrenOpts,
) )
if err != nil { if err != nil {
return fmt.Errorf("creating child processes: %w", err) return fmt.Errorf("creating child processes: %w", err)
@ -282,10 +271,12 @@ func (d *Daemon) reload(
// whatever is in garage // whatever is in garage
newBootstrap.Hosts[thisHost.Name] = thisHost newBootstrap.Hosts[thisHost.Name] = thisHost
diff, err := calcBootstrapDiff(d.daemonConfig, currBootstrap, newBootstrap) diff, err := children.CalculateReloadDiff(
d.daemonConfig, currBootstrap, newBootstrap,
)
if err != nil { if err != nil {
return fmt.Errorf("calculating diff between bootstraps: %w", err) return fmt.Errorf("calculating diff between bootstraps: %w", err)
} else if diff == (bootstrapDiff{}) { } else if diff == (children.ReloadDiff{}) {
d.logger.Info(ctx, "No changes to bootstrap detected") d.logger.Info(ctx, "No changes to bootstrap detected")
return nil return nil
} }
@ -295,29 +286,11 @@ func (d *Daemon) reload(
d.currBootstrap = newBootstrap d.currBootstrap = newBootstrap
d.l.Unlock() d.l.Unlock()
var errs []error if err := d.children.Reload(ctx, newBootstrap, diff); err != nil {
return fmt.Errorf("reloading child processes (diff:%+v): %w", diff, err)
// TODO each of these changed cases should block until its respective
// service is confirmed to have come back online.
// TODO it's possible that reload could be called concurrently, and one call
// would override the reloading done by the other.
if diff.dnsChanged {
d.logger.Info(ctx, "Restarting dnsmasq to account for bootstrap changes")
if err := d.children.RestartDNSMasq(newBootstrap); err != nil {
errs = append(errs, fmt.Errorf("restarting dnsmasq: %w", err))
}
} }
if diff.nebulaChanged { return nil
d.logger.Info(ctx, "Restarting nebula to account for bootstrap changes")
if err := d.children.RestartNebula(newBootstrap); err != nil {
errs = append(errs, fmt.Errorf("restarting nebula: %w", err))
}
}
return errors.Join(errs...)
} }
func (d *Daemon) postInit(ctx context.Context) error { func (d *Daemon) postInit(ctx context.Context) error {

76
go/daemon/garage.go Normal file
View File

@ -0,0 +1,76 @@
package daemon
import (
"context"
"isle/bootstrap"
"isle/daemon/daecommon"
"net"
"strconv"
"isle/garage"
"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
)
func garageAdminClientLogger(logger *mlog.Logger) *mlog.Logger {
return logger.WithNamespace("garageAdminClient")
}
// newGarageAdminClient will return an AdminClient for a local garage instance,
// or it will _panic_ if there is no local instance configured.
func newGarageAdminClient(
logger *mlog.Logger,
daemonConfig daecommon.Config,
adminToken string,
hostBootstrap bootstrap.Bootstrap,
) *garage.AdminClient {
thisHost := hostBootstrap.ThisHost()
return garage.NewAdminClient(
garageAdminClientLogger(logger),
net.JoinHostPort(
thisHost.IP().String(),
strconv.Itoa(daemonConfig.Storage.Allocations[0].AdminPort),
),
adminToken,
)
}
func garageApplyLayout(
ctx context.Context,
logger *mlog.Logger,
daemonConfig daecommon.Config,
adminToken string,
hostBootstrap bootstrap.Bootstrap,
) error {
var (
adminClient = newGarageAdminClient(
logger, daemonConfig, adminToken, hostBootstrap,
)
thisHost = hostBootstrap.ThisHost()
hostName = thisHost.Name
allocs = daemonConfig.Storage.Allocations
peers = make([]garage.PeerLayout, len(allocs))
)
for i, alloc := range allocs {
id := daecommon.BootstrapGarageHostForAlloc(thisHost, alloc).ID
zone := string(hostName)
if alloc.Zone != "" {
zone = alloc.Zone
}
peers[i] = garage.PeerLayout{
ID: id,
Capacity: alloc.Capacity * 1_000_000_000,
Zone: zone,
Tags: []string{},
}
}
return adminClient.ApplyLayout(ctx, peers)
}

View File

@ -1,7 +1,6 @@
package daemon package daemon
import ( import (
"context"
"crypto/rand" "crypto/rand"
"encoding/hex" "encoding/hex"
"errors" "errors"
@ -9,33 +8,8 @@ import (
"io/fs" "io/fs"
"os" "os"
"path/filepath" "path/filepath"
"time"
"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
) )
// until keeps trying fn until it returns nil, returning true. If the context is
// canceled then until returns false.
func until(
ctx context.Context,
logger *mlog.Logger,
descr string,
fn func(context.Context) error,
) bool {
for {
logger.Info(ctx, descr)
err := fn(ctx)
if err == nil {
return true
} else if ctxErr := ctx.Err(); ctxErr != nil {
return false
}
logger.Warn(ctx, descr+" failed, retrying in one second", err)
time.Sleep(1 * time.Second)
}
}
func randStr(l int) string { func randStr(l int) string {
b := make([]byte, l) b := make([]byte, l)
if _, err := rand.Read(b); err != nil { if _, err := rand.Read(b); err != nil {