Implement JoinNetwork RPC method, and accompanying sub-command

This commit is contained in:
Brian Picciano 2024-07-07 12:44:49 +02:00
parent 81368821b7
commit 7d8b274445
12 changed files with 273 additions and 160 deletions

View File

@ -103,16 +103,29 @@ func FromFile(path string) (Bootstrap, error) {
defer f.Close() defer f.Close()
var b Bootstrap var b Bootstrap
if err := json.NewDecoder(f).Decode(&b); err != nil { if err := json.NewDecoder(f).Decode(&b); err != nil {
return Bootstrap{}, fmt.Errorf("decoding json: %w", err) return Bootstrap{}, fmt.Errorf("decoding json: %w", err)
} }
if b.HostAssigned, err = b.SignedHostAssigned.UnwrapUnsafe(); err != nil { return b, nil
return Bootstrap{}, fmt.Errorf("unwrapping host assigned: %w", err)
} }
return b, nil func (b *Bootstrap) UnmarshalJSON(data []byte) error {
type inner Bootstrap
err := json.Unmarshal(data, (*inner)(b))
if err != nil {
return err
}
b.HostAssigned, err = b.SignedHostAssigned.Unwrap(
b.CAPublicCredentials.SigningKey,
)
if err != nil {
return fmt.Errorf("unwrapping HostAssigned: %w", err)
}
return nil
} }
// WriteTo writes the Bootstrap as a new bootstrap to the given io.Writer. // WriteTo writes the Bootstrap as a new bootstrap to the given io.Writer.
@ -123,7 +136,6 @@ func (b Bootstrap) WriteTo(into io.Writer) error {
// ThisHost is a shortcut for b.Hosts[b.HostName], but will panic if the // ThisHost is a shortcut for b.Hosts[b.HostName], but will panic if the
// HostName isn't found in the Hosts map. // HostName isn't found in the Hosts map.
func (b Bootstrap) ThisHost() Host { func (b Bootstrap) ThisHost() Host {
host, ok := b.Hosts[b.Name] host, ok := b.Hosts[b.Name]
if !ok { if !ok {
panic(fmt.Sprintf("hostname %q not defined in bootstrap's hosts", b.Name)) panic(fmt.Sprintf("hostname %q not defined in bootstrap's hosts", b.Name))

View File

@ -78,5 +78,9 @@ type Host struct {
// This assumes that the Host and its data has already been verified against the // This assumes that the Host and its data has already been verified against the
// CA signing key. // CA signing key.
func (h Host) IP() net.IP { func (h Host) IP() net.IP {
return h.PublicCredentials.Cert.Unwrap().Details.Ips[0].IP cert := h.PublicCredentials.Cert.Unwrap()
if len(cert.Details.Ips) == 0 {
panic(fmt.Sprintf("host %q not configured with any ips: %+v", h.Name, h))
}
return cert.Details.Ips[0].IP
} }

View File

@ -2,15 +2,11 @@ package main
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"io/fs"
"os" "os"
"isle/bootstrap"
"isle/daemon" "isle/daemon"
"dev.mediocregopher.com/mediocre-go-lib.git/mctx"
"dev.mediocregopher.com/mediocre-go-lib.git/mlog" "dev.mediocregopher.com/mediocre-go-lib.git/mlog"
) )
@ -31,11 +27,6 @@ var subCmdDaemon = subCmd{
"Write the default configuration file to stdout and exit.", "Write the default configuration file to stdout and exit.",
) )
bootstrapPath := flags.StringP(
"bootstrap-path", "b", "",
`Path to a bootstrap.json file. This only needs to be provided the first time the daemon is started, after that it is ignored. If the isle binary has a bootstrap built into it then this argument is always optional.`,
)
logLevelStr := flags.StringP( logLevelStr := flags.StringP(
"log-level", "l", "info", "log-level", "l", "info",
`Maximum log level which should be output. Values can be "debug", "info", "warn", "error", "fatal". Does not apply to sub-processes`, `Maximum log level which should be output. Values can be "debug", "info", "warn", "error", "fatal". Does not apply to sub-processes`,
@ -64,72 +55,17 @@ var subCmdDaemon = subCmd{
} }
defer runtimeDirCleanup() defer runtimeDirCleanup()
var (
bootstrapStateDirPath = bootstrap.StateDirPath(daemonEnvVars.StateDirPath)
bootstrapAppDirPath = bootstrap.AppDirPath(envAppDirPath)
hostBootstrapPath string
hostBootstrap bootstrap.Bootstrap
)
tryLoadBootstrap := func(path string) bool {
ctx := mctx.Annotate(ctx, "bootstrapFilePath", path)
if err != nil {
return false
} else if hostBootstrap, err = bootstrap.FromFile(path); errors.Is(err, fs.ErrNotExist) {
logger.WarnString(ctx, "bootstrap file not found")
err = nil
return false
} else if err != nil {
err = fmt.Errorf("parsing bootstrap.json at %q: %w", path, err)
return false
}
logger.Info(ctx, "bootstrap file found")
hostBootstrapPath = path
return true
}
switch {
case tryLoadBootstrap(bootstrapStateDirPath):
case *bootstrapPath != "" && tryLoadBootstrap(*bootstrapPath):
case tryLoadBootstrap(bootstrapAppDirPath):
case err != nil:
return fmt.Errorf("attempting to load bootstrap.json file: %w", err)
default:
return errors.New("No bootstrap.json file could be found, and one is not provided with --bootstrap-path")
}
if hostBootstrapPath != bootstrapStateDirPath {
// If the bootstrap file is not being stored in the data dir, copy
// it there, so it can be loaded from there next time.
if err := writeBootstrapToStateDir(hostBootstrap); err != nil {
return fmt.Errorf("writing bootstrap.json to data dir: %w", err)
}
}
daemonConfig, err := daemon.LoadConfig(envAppDirPath, *daemonConfigPath) daemonConfig, err := daemon.LoadConfig(envAppDirPath, *daemonConfigPath)
if err != nil { if err != nil {
return fmt.Errorf("loading daemon config: %w", err) return fmt.Errorf("loading daemon config: %w", err)
} }
// we update this Host's data using whatever configuration has been daemonInst, err := daemon.NewDaemon(
// provided by the daemon config. This way the daemon has the most logger, daemonConfig, envBinDirPath, nil,
// up-to-date possible bootstrap. This updated bootstrap will later get
// updated in garage as a background daemon task, so other hosts will
// see it as well.
if hostBootstrap, err = coalesceDaemonConfigAndBootstrap(hostBootstrap, daemonConfig); err != nil {
return fmt.Errorf("merging daemon config into bootstrap data: %w", err)
}
daemonInst := daemon.NewDaemon(
logger, daemonConfig, envBinDirPath, hostBootstrap, nil,
) )
if err != nil {
return fmt.Errorf("starting daemon: %w", err)
}
defer func() { defer func() {
logger.Info(ctx, "Stopping child processes") logger.Info(ctx, "Stopping child processes")
if err := daemonInst.Shutdown(); err != nil { if err := daemonInst.Shutdown(); err != nil {

View File

@ -1,7 +1,6 @@
package main package main
import ( import (
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"isle/bootstrap" "isle/bootstrap"
@ -30,17 +29,12 @@ var subCmdHostsList = subCmd{
ctx := subCmdCtx.ctx ctx := subCmdCtx.ctx
var resRaw json.RawMessage var res daemon.GetHostsResult
err := subCmdCtx.daemonRCPClient.Call(ctx, &resRaw, "GetHosts", nil) err := subCmdCtx.daemonRCPClient.Call(ctx, &res, "GetHosts", nil)
if err != nil { if err != nil {
return fmt.Errorf("calling GetHosts: %w", err) return fmt.Errorf("calling GetHosts: %w", err)
} }
var res daemon.GetHostsResult
if err := json.Unmarshal(resRaw, &res); err != nil {
return fmt.Errorf("unmarshaling %s into %T: %w", string(resRaw), res, err)
}
type host struct { type host struct {
Name string Name string
VPN struct { VPN struct {

View File

@ -66,6 +66,7 @@ func main() {
subCmdGarage, subCmdGarage,
subCmdHosts, subCmdHosts,
subCmdNebula, subCmdNebula,
subCmdNetwork,
subCmdVersion, subCmdVersion,
) )

View File

@ -0,0 +1,51 @@
package main
import (
"errors"
"fmt"
"isle/bootstrap"
)
var subCmdNetworkJoin = subCmd{
name: "join",
descr: "Joins this host to an existing network",
do: func(subCmdCtx subCmdCtx) error {
var (
ctx = subCmdCtx.ctx
flags = subCmdCtx.flagSet(false)
)
bootstrapPath := flags.StringP(
"bootstrap-path", "b", "", "Path to a bootstrap.json file.",
)
if err := flags.Parse(subCmdCtx.args); err != nil {
return fmt.Errorf("parsing flags: %w", err)
}
if *bootstrapPath == "" {
return errors.New("--bootstrap-path is required")
}
newBootstrap, err := bootstrap.FromFile(*bootstrapPath)
if err != nil {
return fmt.Errorf(
"loading bootstrap from %q: %w", *bootstrapPath, err,
)
}
return subCmdCtx.daemonRCPClient.Call(
ctx, nil, "JoinNetwork", newBootstrap,
)
},
}
var subCmdNetwork = subCmd{
name: "network",
descr: "Sub-commands related to network membership",
do: func(subCmdCtx subCmdCtx) error {
return subCmdCtx.doSubCmd(
subCmdNetworkJoin,
)
},
}

View File

@ -1,32 +1,14 @@
package daemon package daemon
import ( import (
"errors"
"fmt" "fmt"
"io/fs"
"os" "os"
"path/filepath" "path/filepath"
"isle/bootstrap" "isle/bootstrap"
"isle/garage/garagesrv"
) )
func loadHostBootstrap(stateDirPath string) (bootstrap.Bootstrap, error) {
path := bootstrap.StateDirPath(stateDirPath)
hostBootstrap, err := bootstrap.FromFile(path)
if errors.Is(err, fs.ErrNotExist) {
return bootstrap.Bootstrap{}, fmt.Errorf(
"%q not found, has the daemon ever been run?",
stateDirPath,
)
} else if err != nil {
return bootstrap.Bootstrap{}, fmt.Errorf("loading %q: %w", stateDirPath, err)
}
return hostBootstrap, nil
}
func writeBootstrapToStateDir( func writeBootstrapToStateDir(
stateDirPath string, hostBootstrap bootstrap.Bootstrap, stateDirPath string, hostBootstrap bootstrap.Bootstrap,
) error { ) error {
@ -48,3 +30,43 @@ func writeBootstrapToStateDir(
return hostBootstrap.WriteTo(f) return hostBootstrap.WriteTo(f)
} }
func coalesceDaemonConfigAndBootstrap(
daemonConfig Config, hostBootstrap bootstrap.Bootstrap,
) (
bootstrap.Bootstrap, error,
) {
host := bootstrap.Host{
HostAssigned: hostBootstrap.HostAssigned,
HostConfigured: bootstrap.HostConfigured{
Nebula: bootstrap.NebulaHost{
PublicAddr: daemonConfig.VPN.PublicAddr,
},
},
}
if allocs := daemonConfig.Storage.Allocations; len(allocs) > 0 {
for i, alloc := range allocs {
id, rpcPort, err := garagesrv.InitAlloc(alloc.MetaPath, alloc.RPCPort)
if err != nil {
return bootstrap.Bootstrap{}, fmt.Errorf(
"initializing alloc at %q: %w", alloc.MetaPath, err,
)
}
host.Garage.Instances = append(host.Garage.Instances, bootstrap.GarageHostInstance{
ID: id,
RPCPort: rpcPort,
S3APIPort: alloc.S3APIPort,
})
allocs[i].RPCPort = rpcPort
}
}
hostBootstrap.Hosts[host.Name] = host
return hostBootstrap, nil
}

View File

@ -8,6 +8,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"io/fs"
"isle/bootstrap" "isle/bootstrap"
"os" "os"
"sync" "sync"
@ -20,6 +21,13 @@ import (
// with isle, typically via the unix socket. // with isle, typically via the unix socket.
type Daemon interface { type Daemon interface {
// JoinNetwork joins the Daemon to an existing network using the given
// Bootstrap.
//
// Errors:
// - ErrAlreadyJoined
JoinNetwork(context.Context, bootstrap.Bootstrap) error
// GetGarageBootstrapHosts loads (and verifies) the <hostname>.json.signed // GetGarageBootstrapHosts loads (and verifies) the <hostname>.json.signed
// file for all hosts stored in garage. // file for all hosts stored in garage.
GetGarageBootstrapHosts( GetGarageBootstrapHosts(
@ -67,7 +75,8 @@ func (o *Opts) withDefaults() *Opts {
} }
const ( const (
daemonStateInitializing = iota daemonStateNoNetwork = iota
daemonStateInitializing
daemonStateOk daemonStateOk
daemonStateRestarting daemonStateRestarting
daemonStateShutdown daemonStateShutdown
@ -79,13 +88,13 @@ type daemon struct {
envBinDirPath string envBinDirPath string
opts *Opts opts *Opts
l sync.Mutex l sync.RWMutex
state int state int
children *Children children *Children
currBootstrap bootstrap.Bootstrap currBootstrap bootstrap.Bootstrap
cancelFn context.CancelFunc shutdownCh chan struct{}
stoppedCh chan struct{} wg sync.WaitGroup
} }
// NewDaemon initializes and returns a Daemon instance which will manage all // NewDaemon initializes and returns a Daemon instance which will manage all
@ -110,43 +119,89 @@ func NewDaemon(
logger *mlog.Logger, logger *mlog.Logger,
daemonConfig Config, daemonConfig Config,
envBinDirPath string, envBinDirPath string,
currBootstrap bootstrap.Bootstrap,
opts *Opts, opts *Opts,
) Daemon { ) (
ctx, cancelFn := context.WithCancel(context.Background()) Daemon, error,
) {
d := &daemon{ var (
d = &daemon{
logger: logger, logger: logger,
daemonConfig: daemonConfig, daemonConfig: daemonConfig,
envBinDirPath: envBinDirPath, envBinDirPath: envBinDirPath,
opts: opts.withDefaults(), opts: opts.withDefaults(),
currBootstrap: currBootstrap, shutdownCh: make(chan struct{}),
cancelFn: cancelFn, }
stoppedCh: make(chan struct{}), bootstrapFilePath = bootstrap.StateDirPath(d.opts.EnvVars.StateDirPath)
)
currBootstrap, err := bootstrap.FromFile(bootstrapFilePath)
if errors.Is(err, fs.ErrNotExist) {
// daemon has never had a network created or joined
} else if err != nil {
return nil, fmt.Errorf(
"loading bootstrap from %q: %w", bootstrapFilePath, err,
)
} else if err := d.initialize(currBootstrap); err != nil {
return nil, fmt.Errorf("initializing with bootstrap: %w", err)
} }
return d, nil
}
func (d *daemon) initialize(currBootstrap bootstrap.Bootstrap) error {
// we update this Host's data using whatever configuration has been provided
// by the daemon config. This way the daemon has the most up-to-date
// possible bootstrap. This updated bootstrap will later get updated in
// garage as a background daemon task, so other hosts will see it as well.
currBootstrap, err := coalesceDaemonConfigAndBootstrap(
d.daemonConfig, currBootstrap,
)
if err != nil {
return fmt.Errorf("combining daemon configuration into bootstrap: %w", err)
}
err = writeBootstrapToStateDir(d.opts.EnvVars.StateDirPath, currBootstrap)
if err != nil {
return fmt.Errorf("writing bootstrap to state dir: %w", err)
}
d.currBootstrap = currBootstrap
d.state = daemonStateInitializing
ctx, cancel := context.WithCancel(context.Background())
d.wg.Add(1)
go func() { go func() {
d.restartLoop(ctx) defer d.wg.Done()
d.logger.Debug(ctx, "DaemonRestarter stopped") <-d.shutdownCh
close(d.stoppedCh) cancel()
}() }()
return d d.wg.Add(1)
go func() {
defer d.wg.Done()
d.restartLoop(ctx)
d.logger.Debug(ctx, "Daemon restart loop stopped")
}()
return nil
} }
func withInnerChildren[Res any]( func withCurrBootstrap[Res any](
d *daemon, fn func(*Children) (Res, error), d *daemon, fn func(bootstrap.Bootstrap) (Res, error),
) (Res, error) { ) (Res, error) {
var zero Res var zero Res
d.l.Lock() d.l.RLock()
children, state := d.children, d.state defer d.l.RUnlock()
d.l.Unlock()
currBootstrap, state := d.currBootstrap, d.state
switch state { switch state {
case daemonStateNoNetwork:
return zero, ErrNoNetwork
case daemonStateInitializing: case daemonStateInitializing:
return zero, ErrInitializing return zero, ErrInitializing
case daemonStateOk: case daemonStateOk:
return fn(children) return fn(currBootstrap)
case daemonStateRestarting: case daemonStateRestarting:
return zero, ErrRestarting return zero, ErrRestarting
case daemonStateShutdown: case daemonStateShutdown:
@ -167,7 +222,7 @@ func (d *daemon) checkBootstrap(
thisHost := hostBootstrap.ThisHost() thisHost := hostBootstrap.ThisHost()
newHosts, err := d.getGarageBootstrapHosts(ctx) newHosts, err := getGarageBootstrapHosts(ctx, d.logger, hostBootstrap)
if err != nil { if err != nil {
return bootstrap.Bootstrap{}, false, fmt.Errorf("getting hosts from garage: %w", err) return bootstrap.Bootstrap{}, false, fmt.Errorf("getting hosts from garage: %w", err)
} }
@ -233,19 +288,6 @@ func (d *daemon) watchForChanges(ctx context.Context) bootstrap.Bootstrap {
} }
func (d *daemon) restartLoop(ctx context.Context) { func (d *daemon) restartLoop(ctx context.Context) {
defer func() {
d.l.Lock()
d.state = daemonStateShutdown
children := d.children
d.l.Unlock()
if children != nil {
if err := children.Shutdown(); err != nil {
d.logger.Fatal(ctx, "Failed to cleanly shutdown daemon children, there may be orphaned child processes", err)
}
}
}()
wait := func(d time.Duration) bool { wait := func(d time.Duration) bool {
select { select {
case <-ctx.Done(): case <-ctx.Done():
@ -328,18 +370,46 @@ func (d *daemon) restartLoop(ctx context.Context) {
} }
} }
func (d *daemon) JoinNetwork(
ctx context.Context, newBootstrap bootstrap.Bootstrap,
) error {
d.l.Lock()
defer d.l.Unlock()
if d.state != daemonStateNoNetwork {
return ErrAlreadyJoined
}
return d.initialize(newBootstrap)
}
func (d *daemon) GetGarageBootstrapHosts( func (d *daemon) GetGarageBootstrapHosts(
ctx context.Context, ctx context.Context,
) ( ) (
map[string]bootstrap.Host, error, map[string]bootstrap.Host, error,
) { ) {
return withInnerChildren(d, func(*Children) (map[string]bootstrap.Host, error) { return withCurrBootstrap(d, func(
return d.getGarageBootstrapHosts(ctx) currBootstrap bootstrap.Bootstrap,
) (
map[string]bootstrap.Host, error,
) {
return getGarageBootstrapHosts(ctx, d.logger, currBootstrap)
}) })
} }
func (d *daemon) Shutdown() error { func (d *daemon) Shutdown() error {
d.cancelFn() d.l.Lock()
<-d.stoppedCh defer d.l.Unlock()
close(d.shutdownCh)
d.wg.Wait()
d.state = daemonStateShutdown
if d.children != nil {
if err := d.children.Shutdown(); err != nil {
return fmt.Errorf("shutting down children: %w", err)
}
}
return nil return nil
} }

View File

@ -3,11 +3,19 @@ package daemon
import "isle/daemon/jsonrpc2" import "isle/daemon/jsonrpc2"
var ( var (
// ErrNoNetwork is returned when the daemon has never been configured with a
// network.
ErrNoNetwork = jsonrpc2.NewError(1, "No network configured")
// ErrInitializing is returned when a network is unavailable due to still // ErrInitializing is returned when a network is unavailable due to still
// being initialized. // being initialized.
ErrInitializing = jsonrpc2.NewError(1, "Network is being initialized") ErrInitializing = jsonrpc2.NewError(2, "Network is being initialized")
// ErrRestarting is returned when a network is unavailable due to being // ErrRestarting is returned when a network is unavailable due to being
// restarted. // restarted.
ErrRestarting = jsonrpc2.NewError(2, "Network is being restarted") ErrRestarting = jsonrpc2.NewError(3, "Network is being restarted")
// ErrAlreadyJoined is returned when the daemon is instructed to create or
// join a new network, but it is already joined to a network.
ErrAlreadyJoined = jsonrpc2.NewError(4, "Already joined to a network")
) )

View File

@ -11,6 +11,7 @@ import (
"path/filepath" "path/filepath"
"dev.mediocregopher.com/mediocre-go-lib.git/mctx" "dev.mediocregopher.com/mediocre-go-lib.git/mctx"
"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
"github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7"
) )
@ -65,13 +66,13 @@ func (d *daemon) putGarageBoostrapHost(ctx context.Context) error {
return nil return nil
} }
func (d *daemon) getGarageBootstrapHosts( func getGarageBootstrapHosts(
ctx context.Context, ctx context.Context, logger *mlog.Logger, currBootstrap bootstrap.Bootstrap,
) ( ) (
map[string]bootstrap.Host, error, map[string]bootstrap.Host, error,
) { ) {
var ( var (
b = d.currBootstrap b = currBootstrap
client = b.GlobalBucketS3APIClient() client = b.GlobalBucketS3APIClient()
hosts = map[string]bootstrap.Host{} hosts = map[string]bootstrap.Host{}
@ -106,13 +107,13 @@ func (d *daemon) getGarageBootstrapHosts(
obj.Close() obj.Close()
if err != nil { if err != nil {
d.logger.Warn(ctx, "Object contains invalid json", err) logger.Warn(ctx, "Object contains invalid json", err)
continue continue
} }
host, err := authedHost.Unwrap(b.CAPublicCredentials) host, err := authedHost.Unwrap(b.CAPublicCredentials)
if err != nil { if err != nil {
d.logger.Warn(ctx, "Host could not be authenticated", err) logger.Warn(ctx, "Host could not be authenticated", err)
} }
hosts[host.Name] = host hosts[host.Name] = host

View File

@ -26,6 +26,15 @@ func NewRPC(daemon Daemon) *RPC {
return &RPC{daemon} return &RPC{daemon}
} }
// JoinNetwork passes through to the Daemon method of the same name.
func (r *RPC) JoinNetwork(
ctx context.Context, req bootstrap.Bootstrap,
) (
struct{}, error,
) {
return struct{}{}, r.daemon.JoinNetwork(ctx, req)
}
// GetHosts returns all hosts known to the network, sorted by their name. // GetHosts returns all hosts known to the network, sorted by their name.
func (r *RPC) GetHosts( func (r *RPC) GetHosts(
ctx context.Context, req struct{}, ctx context.Context, req struct{},

View File

@ -60,10 +60,9 @@ EOF
isle daemon -l debug --config-path daemon.yml >daemon.log 2>&1 & isle daemon -l debug --config-path daemon.yml >daemon.log 2>&1 &
pid="$!" pid="$!"
echo "Waiting for primus daemon (process $pid) to initialize"
$SHELL "$UTILS/register-cleanup.sh" "$pid" "1-data-1-empty-node-network/primus" $SHELL "$UTILS/register-cleanup.sh" "$pid" "1-data-1-empty-node-network/primus"
echo "Waiting for primus daemon (process $pid) to initialize"
while ! isle hosts list >/dev/null; do sleep 1; done while ! isle hosts list >/dev/null; do sleep 1; done
echo "Creating secondus bootstrap" echo "Creating secondus bootstrap"
@ -82,11 +81,17 @@ EOF
device: isle-secondus device: isle-secondus
EOF EOF
isle daemon -l debug -c daemon.yml -b "$secondus_bootstrap" >daemon.log 2>&1 & isle daemon -l debug -c daemon.yml >daemon.log 2>&1 &
pid="$!" pid="$!"
echo "Waiting for secondus daemon (process $!) to initialize"
$SHELL "$UTILS/register-cleanup.sh" "$pid" "1-data-1-empty-node-network/secondus" $SHELL "$UTILS/register-cleanup.sh" "$pid" "1-data-1-empty-node-network/secondus"
echo "Waiting for secondus daemon (process $!) to start"
while ! [ -e "$ISLE_DAEMON_HTTP_SOCKET_PATH" ]; do sleep 1; done
echo "Joining secondus to the network"
isle network join -b "$secondus_bootstrap"
echo "Waiting for secondus daemon to join"
while ! isle hosts list >/dev/null; do sleep 1; done while ! isle hosts list >/dev/null; do sleep 1; done
) )
fi fi