From 05e91cd657a91f43d50501727f9e961c7345eada Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Mon, 24 Jun 2024 18:55:36 +0200 Subject: [PATCH] Move daemon restarting logic into daemon package --- go/cmd/entrypoint/admin.go | 2 +- go/cmd/entrypoint/daemon.go | 185 ++----------- go/daemon/bootstrap.go | 50 ++++ go/daemon/child_pmux.go | 95 +++++++ go/daemon/daemon.go | 129 ++------- go/daemon/daemon_restarter.go | 262 ++++++++++++++++++ go/daemon/errors.go | 13 + .../utils/with-1-data-1-empty-node-cluster.sh | 4 +- 8 files changed, 475 insertions(+), 265 deletions(-) create mode 100644 go/daemon/bootstrap.go create mode 100644 go/daemon/child_pmux.go create mode 100644 go/daemon/daemon_restarter.go create mode 100644 go/daemon/errors.go diff --git a/go/cmd/entrypoint/admin.go b/go/cmd/entrypoint/admin.go index f522070..ca3262b 100644 --- a/go/cmd/entrypoint/admin.go +++ b/go/cmd/entrypoint/admin.go @@ -199,7 +199,7 @@ var subCmdAdminCreateNetwork = subCmd{ return fmt.Errorf("initializing garage shared global bucket: %w", err) } - if err := daemonInst.Shutdown(ctx); err != nil { + if err := daemonInst.Shutdown(); err != nil { return fmt.Errorf("shutting down daemon: %w (this can mean there are zombie children leftover)", err) } diff --git a/go/cmd/entrypoint/daemon.go b/go/cmd/entrypoint/daemon.go index db5253b..32f1c91 100644 --- a/go/cmd/entrypoint/daemon.go +++ b/go/cmd/entrypoint/daemon.go @@ -1,13 +1,11 @@ package main import ( - "bytes" "context" "errors" "fmt" "io/fs" "os" - "time" "isle/bootstrap" "isle/daemon" @@ -16,151 +14,6 @@ import ( "dev.mediocregopher.com/mediocre-go-lib.git/mlog" ) -// The daemon sub-command deals with starting an actual isle daemon -// process, which is required to be running for most other Isle -// functionality. The sub-command does the following: -// -// * Creates and locks the runtime directory. -// -// * Creates the data directory and copies the appdir bootstrap file into there, -// if it's not already there. -// -// * Merges daemon configuration into the bootstrap configuration, and rewrites -// the bootstrap file. -// -// * Sets up environment variables that all other sub-processes then use, based -// on the runtime dir. -// -// * Dynamically creates the root pmux config and runs pmux. -// -// * (On exit) cleans up the runtime directory. - -// creates a new bootstrap file using available information from the network. If -// the new bootstrap file is different than the existing one, the existing one -// is overwritten and true is returned. -func reloadBootstrap( - ctx context.Context, - logger *mlog.Logger, - daemonInst daemon.Daemon, - hostBootstrap bootstrap.Bootstrap, -) ( - bootstrap.Bootstrap, bool, error, -) { - - thisHost := hostBootstrap.ThisHost() - - newHosts, err := daemonInst.GetGarageBootstrapHosts(ctx) - if err != nil { - return bootstrap.Bootstrap{}, false, fmt.Errorf("getting hosts from garage: %w", err) - } - - // the daemon's view of this host's bootstrap info takes precedence over - // whatever is in garage - newHosts[thisHost.Name] = thisHost - - newHostsHash, err := bootstrap.HostsHash(newHosts) - if err != nil { - return bootstrap.Bootstrap{}, false, fmt.Errorf("calculating hash of new hosts: %w", err) - } - - currHostsHash, err := bootstrap.HostsHash(hostBootstrap.Hosts) - if err != nil { - return bootstrap.Bootstrap{}, false, fmt.Errorf("calculating hash of current hosts: %w", err) - } - - if bytes.Equal(newHostsHash, currHostsHash) { - return hostBootstrap, false, nil - } - - hostBootstrap.Hosts = newHosts - - if err := writeBootstrapToStateDir(hostBootstrap); err != nil { - return bootstrap.Bootstrap{}, false, fmt.Errorf("writing new bootstrap to data dir: %w", err) - } - - return hostBootstrap, true, nil -} - -// runs a single pmux process of daemon, returning only once the env.Context has -// been canceled or bootstrap info has been changed. This will always block -// until the spawned pmux has returned, and returns a copy of hostBootstrap with -// updated boostrap info. -func runDaemonPmuxOnce( - ctx context.Context, - logger *mlog.Logger, - hostBootstrap bootstrap.Bootstrap, - daemonConfig daemon.Config, -) ( - bootstrap.Bootstrap, error, -) { - daemonInst, err := daemon.New( - ctx, - logger.WithNamespace("daemon"), - daemonConfig, - hostBootstrap, - envBinDirPath, - nil, - ) - if err != nil { - return bootstrap.Bootstrap{}, fmt.Errorf("initializing daemon: %w", err) - } - defer func() { - // context.Background() is deliberate here. At this point the entire - // process is shutting down, so whatever owns the process should decide - // when it's been too long. - if err := daemonInst.Shutdown(context.Background()); err != nil { - logger.Error(ctx, "failed to cleanly shutdown daemon", err) - } - }() - - { - logger := logger.WithNamespace("http") - httpSrv, err := newHTTPServer( - ctx, logger, daemon.NewRPC(daemonInst), - ) - if err != nil { - return bootstrap.Bootstrap{}, fmt.Errorf("starting HTTP server: %w", err) - } - defer func() { - // see comment in daemonInst shutdown logic regarding background - // context. - if err := httpSrv.Shutdown(context.Background()); err != nil { - logger.Error(ctx, "Failed to cleanly shutdown http server", err) - } - }() - } - - ticker := time.NewTicker(3 * time.Minute) - defer ticker.Stop() - - for { - select { - - case <-ctx.Done(): - return bootstrap.Bootstrap{}, ctx.Err() - - case <-ticker.C: - - logger.Info(ctx, "checking for changes to bootstrap") - - var ( - changed bool - err error - ) - - if hostBootstrap, changed, err = reloadBootstrap( - ctx, logger, daemonInst, hostBootstrap, - ); err != nil { - return bootstrap.Bootstrap{}, fmt.Errorf("reloading bootstrap: %w", err) - - } else if changed { - fmt.Fprintln(os.Stderr, "bootstrap info has changed, restarting all processes") - return hostBootstrap, nil - } - } - } -} - var subCmdDaemon = subCmd{ name: "daemon", descr: "Runs the isle daemon (Default if no sub-command given)", @@ -274,16 +127,36 @@ var subCmdDaemon = subCmd{ return fmt.Errorf("merging daemon config into bootstrap data: %w", err) } - for { - - hostBootstrap, err = runDaemonPmuxOnce(ctx, logger, hostBootstrap, daemonConfig) - - if errors.Is(err, context.Canceled) { - return nil - - } else if err != nil { - return fmt.Errorf("running pmux for daemon: %w", err) + daemonInst := daemon.NewDaemonRestarter( + logger, daemonConfig, envBinDirPath, hostBootstrap, nil, + ) + defer func() { + logger.Info(ctx, "Stopping child processes") + if err := daemonInst.Shutdown(); err != nil { + logger.Error(ctx, "Shutting down daemon cleanly failed, there may be orphaned child processes", err) } + logger.Info(ctx, "Child processes successfully stopped") + }() + + { + logger := logger.WithNamespace("http") + httpSrv, err := newHTTPServer( + ctx, logger, daemon.NewRPC(daemonInst), + ) + if err != nil { + return fmt.Errorf("starting HTTP server: %w", err) + } + defer func() { + // see comment in daemonInst shutdown logic regarding background + // context. + logger.Info(ctx, "Shutting down HTTP socket") + if err := httpSrv.Shutdown(context.Background()); err != nil { + logger.Error(ctx, "Failed to cleanly shutdown http server", err) + } + }() } + + <-ctx.Done() + return nil }, } diff --git a/go/daemon/bootstrap.go b/go/daemon/bootstrap.go new file mode 100644 index 0000000..6b0118a --- /dev/null +++ b/go/daemon/bootstrap.go @@ -0,0 +1,50 @@ +package daemon + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "isle/bootstrap" +) + +func loadHostBootstrap(stateDirPath string) (bootstrap.Bootstrap, error) { + path := bootstrap.StateDirPath(stateDirPath) + + hostBootstrap, err := bootstrap.FromFile(path) + if errors.Is(err, fs.ErrNotExist) { + return bootstrap.Bootstrap{}, fmt.Errorf( + "%q not found, has the daemon ever been run?", + stateDirPath, + ) + + } else if err != nil { + return bootstrap.Bootstrap{}, fmt.Errorf("loading %q: %w", stateDirPath, err) + } + + return hostBootstrap, nil +} + +func writeBootstrapToStateDir( + stateDirPath string, hostBootstrap bootstrap.Bootstrap, +) error { + var ( + path = bootstrap.StateDirPath(stateDirPath) + dirPath = filepath.Dir(path) + ) + + if err := os.MkdirAll(dirPath, 0700); err != nil { + return fmt.Errorf("creating directory %q: %w", dirPath, err) + } + + f, err := os.Create(path) + if err != nil { + return fmt.Errorf("creating file %q: %w", path, err) + } + + defer f.Close() + + return hostBootstrap.WriteTo(f) +} diff --git a/go/daemon/child_pmux.go b/go/daemon/child_pmux.go new file mode 100644 index 0000000..9270961 --- /dev/null +++ b/go/daemon/child_pmux.go @@ -0,0 +1,95 @@ +package daemon + +import ( + "context" + "fmt" + + "code.betamike.com/micropelago/pmux/pmuxlib" +) + +func (d *daemon) newPmuxConfig() (pmuxlib.Config, error) { + nebulaPmuxProcConfig, err := nebulaPmuxProcConfig( + d.opts.EnvVars.RuntimeDirPath, + d.binDirPath, + d.hostBootstrap, + d.config, + ) + if err != nil { + return pmuxlib.Config{}, fmt.Errorf("generating nebula config: %w", err) + } + + dnsmasqPmuxProcConfig, err := dnsmasqPmuxProcConfig( + d.opts.EnvVars.RuntimeDirPath, + d.binDirPath, + d.hostBootstrap, + d.config, + ) + if err != nil { + return pmuxlib.Config{}, fmt.Errorf( + "generating dnsmasq config: %w", err, + ) + } + + garagePmuxProcConfigs, err := garagePmuxProcConfigs( + d.opts.EnvVars.RuntimeDirPath, d.binDirPath, d.hostBootstrap, d.config, + ) + if err != nil { + return pmuxlib.Config{}, fmt.Errorf( + "generating garage children configs: %w", err, + ) + } + + return pmuxlib.Config{ + Processes: append( + []pmuxlib.ProcessConfig{ + nebulaPmuxProcConfig, + dnsmasqPmuxProcConfig, + }, + garagePmuxProcConfigs..., + ), + }, nil +} + +func (d *daemon) postPmuxInit(ctx context.Context) error { + d.logger.Info(ctx, "waiting for nebula VPN to come online") + if err := waitForNebula(ctx, d.hostBootstrap); err != nil { + return fmt.Errorf("waiting for nebula to start: %w", err) + } + + d.logger.Info(ctx, "waiting for garage instances to come online") + if err := d.waitForGarage(ctx); err != nil { + return fmt.Errorf("waiting for garage to start: %w", err) + } + + if len(d.config.Storage.Allocations) > 0 { + + err := until(ctx, func(ctx context.Context) error { + err := garageApplyLayout(ctx, d.logger, d.hostBootstrap, d.config) + if err != nil { + d.logger.Error(ctx, "applying garage layout", err) + return err + } + + return nil + }) + + if err != nil { + return fmt.Errorf("applying garage layout: %w", err) + } + } + + if !d.opts.SkipHostBootstrapPush { + if err := until(ctx, func(ctx context.Context) error { + if err := d.putGarageBoostrapHost(ctx); err != nil { + d.logger.Error(ctx, "updating host info in garage", err) + return err + } + + return nil + }); err != nil { + return fmt.Errorf("updating host info in garage: %w", err) + } + } + + return nil +} diff --git a/go/daemon/daemon.go b/go/daemon/daemon.go index 16c0992..e736099 100644 --- a/go/daemon/daemon.go +++ b/go/daemon/daemon.go @@ -8,23 +8,11 @@ import ( "io" "isle/bootstrap" "os" - "time" "code.betamike.com/micropelago/pmux/pmuxlib" "dev.mediocregopher.com/mediocre-go-lib.git/mlog" ) -type daemon struct { - logger *mlog.Logger - config Config - hostBootstrap bootstrap.Bootstrap - binDirPath string - opts Opts - - pmuxCancelFn context.CancelFunc - pmuxStoppedCh chan struct{} -} - // Daemon presents all functionality required for client frontends to interact // with isle, typically via the unix socket. type Daemon interface { @@ -38,12 +26,11 @@ type Daemon interface { ) // Shutdown blocks until all resources held or created by the daemon, - // including child processes it has started, have been cleaned up, or until - // the context is canceled. + // including child processes it has started, have been cleaned up. // // If this returns an error then it's possible that child processes are // still running and are no longer managed. - Shutdown(context.Context) error + Shutdown() error } // Opts are optional parameters which can be passed in when initializing a new @@ -80,6 +67,17 @@ func (o *Opts) withDefaults() *Opts { return o } +type daemon struct { + logger *mlog.Logger + config Config + hostBootstrap bootstrap.Bootstrap + binDirPath string + opts Opts + + pmuxCancelFn context.CancelFunc + pmuxStoppedCh chan struct{} +} + // New initialized and returns a Daemon. If initialization fails an error is // returned. func New( @@ -94,43 +92,6 @@ func New( ) { opts = opts.withDefaults() - nebulaPmuxProcConfig, err := nebulaPmuxProcConfig( - opts.EnvVars.RuntimeDirPath, - binDirPath, - hostBootstrap, - config, - ) - if err != nil { - return nil, fmt.Errorf("generating nebula config: %w", err) - } - - dnsmasqPmuxProcConfig, err := dnsmasqPmuxProcConfig( - opts.EnvVars.RuntimeDirPath, - binDirPath, - hostBootstrap, - config, - ) - if err != nil { - return nil, fmt.Errorf("generating dnsmasq config: %w", err) - } - - garagePmuxProcConfigs, err := garagePmuxProcConfigs( - opts.EnvVars.RuntimeDirPath, binDirPath, hostBootstrap, config, - ) - if err != nil { - return nil, fmt.Errorf("generating garage children configs: %w", err) - } - - pmuxConfig := pmuxlib.Config{ - Processes: append( - []pmuxlib.ProcessConfig{ - nebulaPmuxProcConfig, - dnsmasqPmuxProcConfig, - }, - garagePmuxProcConfigs..., - ), - } - pmuxCtx, pmuxCancelFn := context.WithCancel(context.Background()) d := &daemon{ @@ -143,16 +104,20 @@ func New( pmuxStoppedCh: make(chan struct{}), } + pmuxConfig, err := d.newPmuxConfig() + if err != nil { + return nil, fmt.Errorf("generating pmux config: %w", err) + } + go func() { defer close(d.pmuxStoppedCh) pmuxlib.Run(pmuxCtx, d.opts.Stdout, d.opts.Stderr, pmuxConfig) + d.logger.Debug(pmuxCtx, "pmux stopped") }() if initErr := d.postPmuxInit(ctx); initErr != nil { logger.Warn(ctx, "failed to initialize daemon, shutting down child processes", err) - shutdownCtx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) - defer cancel() - if err := d.Shutdown(shutdownCtx); err != nil { + if err := d.Shutdown(); err != nil { panic(fmt.Sprintf( "failed to shut down child processes after initialization"+ " error, there may be zombie children leftover."+ @@ -167,56 +132,8 @@ func New( return d, nil } -func (d *daemon) postPmuxInit(ctx context.Context) error { - d.logger.Info(ctx, "waiting for nebula VPN to come online") - if err := waitForNebula(ctx, d.hostBootstrap); err != nil { - return fmt.Errorf("waiting for nebula to start: %w", err) - } - - d.logger.Info(ctx, "waiting for garage instances to come online") - if err := d.waitForGarage(ctx); err != nil { - return fmt.Errorf("waiting for garage to start: %w", err) - } - - if len(d.config.Storage.Allocations) > 0 { - - err := until(ctx, func(ctx context.Context) error { - err := garageApplyLayout(ctx, d.logger, d.hostBootstrap, d.config) - if err != nil { - d.logger.Error(ctx, "applying garage layout", err) - return err - } - - return nil - }) - - if err != nil { - return fmt.Errorf("applying garage layout: %w", err) - } - } - - if !d.opts.SkipHostBootstrapPush { - if err := until(ctx, func(ctx context.Context) error { - if err := d.putGarageBoostrapHost(ctx); err != nil { - d.logger.Error(ctx, "updating host info in garage", err) - return err - } - - return nil - }); err != nil { - return fmt.Errorf("updating host info in garage: %w", err) - } - } - +func (d *daemon) Shutdown() error { + d.pmuxCancelFn() + <-d.pmuxStoppedCh return nil } - -func (d *daemon) Shutdown(ctx context.Context) error { - d.pmuxCancelFn() - select { - case <-ctx.Done(): - return ctx.Err() - case <-d.pmuxStoppedCh: - return nil - } -} diff --git a/go/daemon/daemon_restarter.go b/go/daemon/daemon_restarter.go new file mode 100644 index 0000000..54ac7f8 --- /dev/null +++ b/go/daemon/daemon_restarter.go @@ -0,0 +1,262 @@ +package daemon + +import ( + "bytes" + "context" + "errors" + "fmt" + "isle/bootstrap" + "sync" + "time" + + "dev.mediocregopher.com/mediocre-go-lib.git/mlog" +) + +const ( + daemonRestarterStateInitializing = iota + daemonRestarterStateOk + daemonRestarterStateRestarting + daemonRestarterStateShutdown +) + +type daemonRestarter struct { + logger *mlog.Logger + daemonConfig Config + envBinDirPath string + opts *Opts + + l sync.Mutex + state int + inner Daemon + currBootstrap bootstrap.Bootstrap + + cancelFn context.CancelFunc + stoppedCh chan struct{} +} + +// NewDaemonRestarter will wrap a Daemon such that it will be automatically +// shutdown and re-created whenever there's changes in the cluster which require +// the configuration to be changed (e.g. a new nebula lighthouse). +// +// While still starting up the daemon for the first time all methods will return +// ErrInitializing, except Shutdown which will block until initialization is +// canceled. +// +// During a restart all methods will return ErrRestarting, except Shutdown which +// will block until the currently executing restart is finished and then +// shutdown cleanly from there. +// +// TODO make daemonRestarter smarter, it currently restarts on _any_ change, but +// it should restart itself only when there's something actually requiring a +// restart. +func NewDaemonRestarter( + logger *mlog.Logger, + daemonConfig Config, + envBinDirPath string, + currBootstrap bootstrap.Bootstrap, + opts *Opts, +) Daemon { + ctx, cancelFn := context.WithCancel(context.Background()) + + dr := &daemonRestarter{ + logger: logger, + daemonConfig: daemonConfig, + envBinDirPath: envBinDirPath, + opts: opts.withDefaults(), + currBootstrap: currBootstrap, + cancelFn: cancelFn, + stoppedCh: make(chan struct{}), + } + + go func() { + dr.restartLoop(ctx) + dr.logger.Debug(ctx, "DaemonRestarter stopped") + close(dr.stoppedCh) + }() + + return dr +} + +func withInnerDaemon[Res any]( + dr *daemonRestarter, fn func(Daemon) (Res, error), +) (Res, error) { + var zero Res + dr.l.Lock() + inner, state := dr.inner, dr.state + dr.l.Unlock() + + switch state { + case daemonRestarterStateInitializing: + return zero, ErrInitializing + case daemonRestarterStateOk: + return fn(inner) + case daemonRestarterStateRestarting: + return zero, ErrRestarting + case daemonRestarterStateShutdown: + return zero, errors.New("already shutdown") + default: + panic(fmt.Sprintf("unknown state %d", dr.state)) + } +} + +// creates a new bootstrap file using available information from the network. If +// the new bootstrap file is different than the existing one, the existing one +// is overwritten and true is returned. +func (dr *daemonRestarter) checkBootstrap( + ctx context.Context, hostBootstrap bootstrap.Bootstrap, +) ( + bootstrap.Bootstrap, bool, error, +) { + + thisHost := hostBootstrap.ThisHost() + + newHosts, err := dr.inner.GetGarageBootstrapHosts(ctx) + if err != nil { + return bootstrap.Bootstrap{}, false, fmt.Errorf("getting hosts from garage: %w", err) + } + + // the daemon's view of this host's bootstrap info takes precedence over + // whatever is in garage + newHosts[thisHost.Name] = thisHost + + newHostsHash, err := bootstrap.HostsHash(newHosts) + if err != nil { + return bootstrap.Bootstrap{}, false, fmt.Errorf("calculating hash of new hosts: %w", err) + } + + currHostsHash, err := bootstrap.HostsHash(hostBootstrap.Hosts) + if err != nil { + return bootstrap.Bootstrap{}, false, fmt.Errorf("calculating hash of current hosts: %w", err) + } + + if bytes.Equal(newHostsHash, currHostsHash) { + return hostBootstrap, false, nil + } + + hostBootstrap.Hosts = newHosts + + return hostBootstrap, true, nil +} + +// blocks until a new bootstrap is available or context is canceled +func (dr *daemonRestarter) watchForChanges(ctx context.Context) bootstrap.Bootstrap { + + ticker := time.NewTicker(3 * time.Minute) + defer ticker.Stop() + + for { + select { + + case <-ctx.Done(): + return bootstrap.Bootstrap{} + + case <-ticker.C: + + dr.logger.Info(ctx, "Checking for changes to bootstrap") + + newBootstrap, changed, err := dr.checkBootstrap( + ctx, dr.currBootstrap, + ) + if err != nil { + dr.logger.Error(ctx, "Checking bootstrap for changes failed", err) + continue + } else if !changed { + continue + } + + err = writeBootstrapToStateDir(dr.opts.EnvVars.StateDirPath, newBootstrap) + if err != nil { + dr.logger.Error(ctx, "Writing new bootstrap to disk failed", err) + continue + } + + return newBootstrap + } + } +} + +func (dr *daemonRestarter) restartLoop(ctx context.Context) { + defer func() { + dr.l.Lock() + dr.state = daemonRestarterStateShutdown + inner := dr.inner + dr.l.Unlock() + + if inner != nil { + if err := inner.Shutdown(); err != nil { + dr.logger.Fatal(ctx, "failed to cleanly shutdown daemon, there may be orphaned child processes", err) + } + } + }() + + wait := func(d time.Duration) bool { + select { + case <-ctx.Done(): + return false + case <-time.After(d): + return true + } + } + + for { + if ctx.Err() != nil { + return + } + + dr.logger.Info(ctx, "Creating new daemon") + daemonInst, err := New( + ctx, + dr.logger.WithNamespace("daemon"), + dr.daemonConfig, + dr.currBootstrap, + dr.envBinDirPath, + dr.opts, + ) + if errors.Is(err, context.Canceled) { + return + } else if err != nil { + dr.logger.Error(ctx, "failed to initialize daemon", err) + if !wait(1 * time.Second) { + return + } + continue + } + + dr.l.Lock() + dr.inner = daemonInst + dr.state = daemonRestarterStateOk + dr.l.Unlock() + + newBootstrap := dr.watchForChanges(ctx) + if ctx.Err() != nil { + return + } + + dr.logger.Info(ctx, "Bootstrap has changed, will restart daemon") + dr.l.Lock() + dr.currBootstrap = newBootstrap + dr.state = daemonRestarterStateRestarting + dr.l.Unlock() + + dr.logger.Info(ctx, "Shutting down previous daemon") + if err := dr.inner.Shutdown(); err != nil { + dr.logger.Fatal(ctx, "failed to cleanly shutdown daemon, there may be orphaned child processes", err) + } + } +} + +func (dr *daemonRestarter) GetGarageBootstrapHosts( + ctx context.Context, +) ( + map[string]bootstrap.Host, error, +) { + return withInnerDaemon(dr, func(inner Daemon) (map[string]bootstrap.Host, error) { + return inner.GetGarageBootstrapHosts(ctx) + }) +} + +func (dr *daemonRestarter) Shutdown() error { + dr.cancelFn() + <-dr.stoppedCh + return nil +} diff --git a/go/daemon/errors.go b/go/daemon/errors.go new file mode 100644 index 0000000..6370216 --- /dev/null +++ b/go/daemon/errors.go @@ -0,0 +1,13 @@ +package daemon + +import "isle/daemon/jsonrpc2" + +var ( + // ErrInitializing is returned when a cluster is unavailable due to still + // being initialized. + ErrInitializing = jsonrpc2.NewError(1, "Cluster is being initialized") + + // ErrRestarting is returned when a cluster is unavailable due to being + // restarted. + ErrRestarting = jsonrpc2.NewError(2, "Cluster is being restarted") +) diff --git a/tests/utils/with-1-data-1-empty-node-cluster.sh b/tests/utils/with-1-data-1-empty-node-cluster.sh index 836bee9..67318fd 100644 --- a/tests/utils/with-1-data-1-empty-node-cluster.sh +++ b/tests/utils/with-1-data-1-empty-node-cluster.sh @@ -58,7 +58,7 @@ EOF --name "testing" \ > admin.json - isle daemon --config-path daemon.yml >daemon.log 2>&1 & + isle daemon -l debug --config-path daemon.yml >daemon.log 2>&1 & pid="$!" echo "Waiting for primus daemon (process $pid) to initialize" @@ -82,7 +82,7 @@ EOF device: isle-secondus EOF - isle daemon -c daemon.yml -b "$secondus_bootstrap" >daemon.log 2>&1 & + isle daemon -l debug -c daemon.yml -b "$secondus_bootstrap" >daemon.log 2>&1 & pid="$!" echo "Waiting for secondus daemon (process $!) to initialize"