isle/go/daemon/daemon.go

// Package daemon implements the isle daemon, which is a long-running service
// managing all isle background tasks and sub-processes for a single network.
package daemon

import (
	"bytes"
	"context"
	"crypto/rand"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"isle/bootstrap"
	"isle/jsonutil"
	"isle/nebula"
	"isle/secrets"
	"net/netip"
	"os"
	"path/filepath"
	"sync"
	"time"

	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
)

// CreateHostOpts are optional parameters to the CreateHost method.
type CreateHostOpts struct {
	// IP address of the new host. An IP address will be randomly chosen if one
	// is not given here.
	IP netip.Addr

	// CanCreateHosts indicates that the bootstrap produced by CreateHost should
	// give the new host the ability to create new hosts as well.
	CanCreateHosts bool
}

// CreateNebulaCertificateOpts are optional parameters to the
// CreateNebulaCertificate method.
type CreateNebulaCertificateOpts struct {

	// IP, if given will be used for the host's IP in the created cert. If this
	// is given then it is not required that the host have an entry in garage.
	//
	// TODO once `hosts create` automatically adds the host to garage this can
	// be removed.
	IP netip.Addr
}

// Daemon presents all functionality required for client frontends to interact
// with isle, typically via the unix socket.
type Daemon interface {

	// CreateNetwork will initialize a new network using the given parameters.
	// - name: Human-readable name of the network.
	// - domain: Primary domain name that network services are served under.
	// - ipNet: An IP subnet, in CIDR form, which will be the overall range of
	//   possible IPs in the network. The first IP in this network range will
	//   become this first host's IP.
	// - hostName: The name of this first host in the network.
	//
	// The daemon on which this is called will become the first host in the
	// network, and will have full administrative privileges.
	CreateNetwork(
		ctx context.Context, name, domain string,
		ipNet nebula.IPNet,
		hostName nebula.HostName,
	) error

	// JoinNetwork joins the Daemon to an existing network using the given
	// Bootstrap.
	//
	// Errors:
	// - ErrAlreadyJoined
	JoinNetwork(context.Context, JoiningBootstrap) error

	// GetBootstraps returns the currently active Bootstrap.
	GetBootstrap(context.Context) (bootstrap.Bootstrap, error)

	// GetGarageClientParams returns a GarageClientParams for the current
	// network state.
	GetGarageClientParams(context.Context) (GarageClientParams, error)

	// RemoveHost removes the host of the given name from the network.
	RemoveHost(context.Context, nebula.HostName) error

	// CreateHost creates a bootstrap for a new host with the given name and IP
	// address.
	CreateHost(
		ctx context.Context,
		hostName nebula.HostName,
		opts CreateHostOpts,
	) (
		JoiningBootstrap, error,
	)

	// CreateNebulaCertificate creates and signs a new nebula certficate for an
	// existing host, given the public key for that host. This is currently
	// mostly useful for creating certs for mobile devices.
	//
	// Errors:
	// - ErrHostNotFound
	CreateNebulaCertificate(
		ctx context.Context,
		hostName nebula.HostName,
		hostPubKey nebula.EncryptingPublicKey,
		opts CreateNebulaCertificateOpts,
	) (
		nebula.Certificate, error,
	)

	// Shutdown blocks until all resources held or created by the daemon,
	// including child processes it has started, have been cleaned up.
	//
	// If this returns an error then it's possible that child processes are
	// still running and are no longer managed.
	Shutdown() error
}

// Opts are optional parameters which can be passed in when initializing a new
// Daemon instance. A nil Opts is equivalent to a zero value.
type Opts struct {
	// Stdout and Stderr are what the associated outputs from child processes
	// will be directed to.
	Stdout, Stderr io.Writer

	EnvVars EnvVars // Defaults to that returned by GetEnvVars.
}

func (o *Opts) withDefaults() *Opts {
	if o == nil {
		o = new(Opts)
	}

	if o.Stdout == nil {
		o.Stdout = os.Stdout
	}

	if o.Stderr == nil {
		o.Stderr = os.Stderr
	}

	if o.EnvVars == (EnvVars{}) {
		o.EnvVars = GetEnvVars()
	}

	return o
}

const (
	daemonStateNoNetwork = iota
	daemonStateInitializing
	daemonStateOk
	daemonStateShutdown
)

type daemon struct {
	logger        *mlog.Logger
	daemonConfig  Config
	envBinDirPath string
	opts          *Opts

	secretsStore     secrets.Store
	garageAdminToken string

	l             sync.RWMutex
	state         int
	children      *Children
	currBootstrap bootstrap.Bootstrap

	shutdownCh chan struct{}
	wg         sync.WaitGroup
}

// NewDaemon initializes and returns a Daemon instance which will manage all
// child processes and state required by the isle service, as well as an HTTP
// socket over which RPC calls will be served.
//
// Inner Children instance(s) will be wrapped such that they will be
// automatically shutdown and re-created whenever there's changes in the network
// which require the configuration to be changed (e.g. a new nebula lighthouse).
// During such an inner restart all methods will return ErrRestarting, except
// Shutdown which will block until the currently executing restart is finished
// and then shutdown cleanly from there.
//
// While still starting up the Daemon for the first time all methods will return
// ErrInitializing, except Shutdown which will block until initialization is
// canceled.
func NewDaemon(
	ctx context.Context,
	logger *mlog.Logger,
	daemonConfig Config,
	envBinDirPath string,
	opts *Opts,
) (
	Daemon, error,
) {
	var (
		d = &daemon{
			logger:           logger,
			daemonConfig:     daemonConfig,
			envBinDirPath:    envBinDirPath,
			opts:             opts.withDefaults(),
			garageAdminToken: randStr(32),
			shutdownCh:       make(chan struct{}),
		}
		bootstrapFilePath = bootstrap.StateDirPath(d.opts.EnvVars.StateDirPath)
	)

	if err := d.opts.EnvVars.init(); err != nil {
		return nil, fmt.Errorf("initializing daemon directories: %w", err)
	}

	var (
		secretsPath = filepath.Join(d.opts.EnvVars.StateDirPath, "secrets")
		err         error
	)

	if d.secretsStore, err = secrets.NewFSStore(secretsPath); err != nil {
		return nil, fmt.Errorf(
			"initializing secrets store at %q: %w", secretsPath, err,
		)
	}

	var currBootstrap bootstrap.Bootstrap
	err = jsonutil.LoadFile(&currBootstrap, bootstrapFilePath)
	if errors.Is(err, fs.ErrNotExist) {
		// daemon has never had a network created or joined
	} else if err != nil {
		return nil, fmt.Errorf(
			"loading bootstrap from %q: %w", bootstrapFilePath, err,
		)
	} else if err := d.initialize(ctx, currBootstrap); err != nil {
		return nil, fmt.Errorf("initializing with bootstrap: %w", err)
	}

	return d, nil
}

// initialize must be called with d.l write lock held, but it will unlock the
// lock itself.
func (d *daemon) initialize(
	ctx context.Context, currBootstrap bootstrap.Bootstrap,
) error {
	defer d.l.Unlock()

	// we update this Host's data using whatever configuration has been provided
	// by the daemon config. This way the daemon has the most up-to-date
	// possible bootstrap. This updated bootstrap will later get updated in
	// garage as a background daemon task, so other hosts will see it as well.
	currBootstrap, err := coalesceDaemonConfigAndBootstrap(
		d.daemonConfig, currBootstrap,
	)
	if err != nil {
		return fmt.Errorf("combining daemon configuration into bootstrap: %w", err)
	}

	err = writeBootstrapToStateDir(d.opts.EnvVars.StateDirPath, currBootstrap)
	if err != nil {
		return fmt.Errorf("writing bootstrap to state dir: %w", err)
	}

	d.currBootstrap = currBootstrap
	d.state = daemonStateInitializing

	d.logger.Info(ctx, "Creating child processes")
	d.children, err = NewChildren(
		ctx,
		d.logger.WithNamespace("children"),
		d.envBinDirPath,
		d.secretsStore,
		d.daemonConfig,
		d.garageAdminToken,
		currBootstrap,
		d.opts,
	)
	if err != nil {
		return fmt.Errorf("creating child processes: %w", err)
	}

	d.logger.Info(ctx, "Child processes created")

	if err := d.postInit(ctx); err != nil {
		d.logger.Error(ctx, "Post-initialization failed, stopping child processes", err)
		d.children.Shutdown()
		return fmt.Errorf("performing post-initialization: %w", err)
	}

	d.state = daemonStateOk

	ctx, cancel := context.WithCancel(context.Background())
	d.wg.Add(1)
	go func() {
		defer d.wg.Done()
		<-d.shutdownCh
		cancel()
	}()

	d.wg.Add(1)
	go func() {
		defer d.wg.Done()
		d.reloadLoop(ctx)
		d.logger.Debug(ctx, "Daemon restart loop stopped")
	}()

	return nil
}

func withCurrBootstrap[Res any](
	d *daemon, fn func(bootstrap.Bootstrap) (Res, error),
) (Res, error) {
	var zero Res
	d.l.RLock()
	defer d.l.RUnlock()

	currBootstrap, state := d.currBootstrap, d.state

	switch state {
	case daemonStateNoNetwork:
		return zero, ErrNoNetwork
	case daemonStateInitializing:
		return zero, ErrInitializing
	case daemonStateOk:
		return fn(currBootstrap)
	case daemonStateShutdown:
		return zero, errors.New("already shutdown")
	default:
		panic(fmt.Sprintf("unknown state %d", d.state))
	}
}

// reload will check the existing hosts data from currBootstrap against a
// potentially updated set of hosts data, and if there are any differences will
// perform whatever changes are necessary.
func (d *daemon) reload(
	ctx context.Context,
	currBootstrap bootstrap.Bootstrap,
	newHosts map[nebula.HostName]bootstrap.Host,
) error {
	var (
		newBootstrap = currBootstrap
		thisHost     = currBootstrap.ThisHost()
	)

	newBootstrap.Hosts = newHosts

	// the daemon's view of this host's bootstrap info takes precedence over
	// whatever is in garage
	newBootstrap.Hosts[thisHost.Name] = thisHost

	diff, err := calcBootstrapDiff(d.daemonConfig, currBootstrap, newBootstrap)
	if err != nil {
		return fmt.Errorf("calculating diff between bootstraps: %w", err)
	} else if diff == (bootstrapDiff{}) {
		d.logger.Info(ctx, "No changes to bootstrap detected")
		return nil
	}

	d.logger.Info(ctx, "Bootstrap has changed, storing new bootstrap")
	d.l.Lock()
	d.currBootstrap = newBootstrap
	d.l.Unlock()

	var errs []error

	// TODO each of these changed cases should block until its respective
	// service is confirmed to have come back online.

	// TODO it's possible that reload could be called concurrently, and one call
	// would override the reloading done by the other.

	if diff.dnsChanged {
		d.logger.Info(ctx, "Restarting dnsmasq to account for bootstrap changes")
		if err := d.children.RestartDNSMasq(newBootstrap); err != nil {
			errs = append(errs, fmt.Errorf("restarting dnsmasq: %w", err))
		}
	}

	if diff.nebulaChanged {
		d.logger.Info(ctx, "Restarting nebula to account for bootstrap changes")
		if err := d.children.RestartNebula(newBootstrap); err != nil {
			errs = append(errs, fmt.Errorf("restarting nebula: %w", err))
		}
	}

	return errors.Join(errs...)
}

func (d *daemon) postInit(ctx context.Context) error {
	if len(d.daemonConfig.Storage.Allocations) > 0 {
		d.logger.Info(ctx, "Applying garage layout")
		if err := garageApplyLayout(
			ctx, d.logger, d.daemonConfig, d.garageAdminToken, d.currBootstrap,
		); err != nil {
			return fmt.Errorf("applying garage layout: %w", err)
		}
	}

	// This is only necessary during network creation, otherwise the bootstrap
	// should already have these credentials built in.
	//
	// TODO this is pretty hacky, but there doesn't seem to be a better way to
	// manage it at the moment.
	_, err := getGarageS3APIGlobalBucketCredentials(ctx, d.secretsStore)
	if errors.Is(err, secrets.ErrNotFound) {
		d.logger.Info(ctx, "Initializing garage shared global bucket")
		garageGlobalBucketCreds, err := garageInitializeGlobalBucket(
			ctx,
			d.logger,
			d.daemonConfig,
			d.garageAdminToken,
			d.currBootstrap,
		)
		if err != nil {
			return fmt.Errorf("initializing global bucket: %w", err)
		}

		err = setGarageS3APIGlobalBucketCredentials(
			ctx, d.secretsStore, garageGlobalBucketCreds,
		)
		if err != nil {
			return fmt.Errorf("storing global bucket creds: %w", err)
		}
	}

	d.logger.Info(ctx, "Updating host info in garage")
	err = d.putGarageBoostrapHost(ctx, d.currBootstrap)
	if err != nil {
		return fmt.Errorf("updating host info in garage: %w", err)
	}

	return nil
}

func (d *daemon) reloadLoop(ctx context.Context) {
	ticker := time.NewTicker(3 * time.Minute)
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			return

		case <-ticker.C:
			d.l.RLock()
			currBootstrap := d.currBootstrap
			d.l.RUnlock()

			d.logger.Info(ctx, "Checking for bootstrap changes")
			newHosts, err := d.getGarageBootstrapHosts(ctx, currBootstrap)
			if err != nil {
				d.logger.Error(ctx, "Failed to get hosts from garage", err)
				continue
			}

			// TODO there's some potential race conditions here, where
			// CreateHost could be called at this point, write the new host to
			// garage and the bootstrap, but then this reload call removes the
			// host from this bootstrap/children until the next reload.

			if err := d.reload(ctx, currBootstrap, newHosts); err != nil {
				d.logger.Error(ctx, "Reloading with new host data failed", err)
				continue
			}
		}
	}
}

func (d *daemon) CreateNetwork(
	ctx context.Context,
	name, domain string, ipNet nebula.IPNet, hostName nebula.HostName,
) error {
	nebulaCACreds, err := nebula.NewCACredentials(domain, ipNet)
	if err != nil {
		return fmt.Errorf("creating nebula CA cert: %w", err)
	}

	var (
		creationParams = bootstrap.CreationParams{
			ID:     randStr(32),
			Name:   name,
			Domain: domain,
		}

		garageRPCSecret = randStr(32)
	)

	err = setGarageRPCSecret(ctx, d.secretsStore, garageRPCSecret)
	if err != nil {
		return fmt.Errorf("setting garage RPC secret: %w", err)
	}

	err = setNebulaCASigningPrivateKey(ctx, d.secretsStore, nebulaCACreds.SigningPrivateKey)
	if err != nil {
		return fmt.Errorf("setting nebula CA signing key secret: %w", err)
	}

	hostBootstrap, err := bootstrap.New(
		nebulaCACreds,
		creationParams,
		map[nebula.HostName]bootstrap.Host{},
		hostName,
		ipNet.FirstAddr(),
	)
	if err != nil {
		return fmt.Errorf("initializing bootstrap data: %w", err)
	}

	d.l.Lock()

	if d.state != daemonStateNoNetwork {
		d.l.Unlock()
		return ErrAlreadyJoined
	}

	if len(d.daemonConfig.Storage.Allocations) < 3 {
		d.l.Unlock()
		return ErrInvalidConfig.WithData(
			"At least three storage allocations are required.",
		)
	}

	// initialize will unlock d.l
	err = d.initialize(ctx, hostBootstrap)
	if err != nil {
		return fmt.Errorf("initializing daemon: %w", err)
	}

	return nil
}

func (d *daemon) JoinNetwork(
	ctx context.Context, newBootstrap JoiningBootstrap,
) error {
	d.l.Lock()

	if d.state != daemonStateNoNetwork {
		d.l.Unlock()
		return ErrAlreadyJoined
	}

	err := secrets.Import(ctx, d.secretsStore, newBootstrap.Secrets)
	if err != nil {
		d.l.Unlock()
		return fmt.Errorf("importing secrets: %w", err)
	}

	// initialize will unlock d.l
	err = d.initialize(ctx, newBootstrap.Bootstrap)
	if err != nil {
		return fmt.Errorf("initializing daemon: %w", err)
	}

	return nil
}

func (d *daemon) GetBootstrap(ctx context.Context) (bootstrap.Bootstrap, error) {
	return withCurrBootstrap(d, func(
		currBootstrap bootstrap.Bootstrap,
	) (
		bootstrap.Bootstrap, error,
	) {
		return currBootstrap, nil
	})
}

func (d *daemon) GetGarageClientParams(
	ctx context.Context,
) (
	GarageClientParams, error,
) {
	return withCurrBootstrap(d, func(
		currBootstrap bootstrap.Bootstrap,
	) (
		GarageClientParams, error,
	) {
		return d.getGarageClientParams(ctx, currBootstrap)
	})
}

func (d *daemon) RemoveHost(ctx context.Context, hostName nebula.HostName) error {
	// TODO RemoveHost should publish a certificate revocation for the host
	// being removed.
	_, err := withCurrBootstrap(d, func(
		currBootstrap bootstrap.Bootstrap,
	) (
		struct{}, error,
	) {
		garageClientParams, err := d.getGarageClientParams(ctx, currBootstrap)
		if err != nil {
			return struct{}{}, fmt.Errorf("get garage client params: %w", err)
		}

		client := garageClientParams.GlobalBucketS3APIClient()
		return struct{}{}, removeGarageBootstrapHost(ctx, client, hostName)
	})
	return err
}

func makeCACreds(
	currBootstrap bootstrap.Bootstrap,
	caSigningPrivateKey nebula.SigningPrivateKey,
) nebula.CACredentials {
	return nebula.CACredentials{
		Public:            currBootstrap.CAPublicCredentials,
		SigningPrivateKey: caSigningPrivateKey,
	}
}

func chooseAvailableIP(b bootstrap.Bootstrap) (netip.Addr, error) {
	var (
		cidrIPNet = b.CAPublicCredentials.Cert.Unwrap().Details.Subnets[0]
		cidrMask  = cidrIPNet.Mask
		cidrIPB   = cidrIPNet.IP

		cidr           = netip.MustParsePrefix(cidrIPNet.String())
		cidrIP         = cidr.Addr()
		cidrSuffixBits = cidrIP.BitLen() - cidr.Bits()

		inUseIPs = make(map[netip.Addr]struct{}, len(b.Hosts))
	)

	for _, host := range b.Hosts {
		inUseIPs[host.IP()] = struct{}{}
	}

	// first check that there are any addresses at all. We can determine the
	// number of possible addresses using the network CIDR. The first IP in a
	// subnet is the network identifier, and is reserved. The last IP is the
	// broadcast IP, and is also reserved. Hence, the -2.
	usableIPs := (1 << cidrSuffixBits) - 2
	if len(inUseIPs) >= usableIPs {
		return netip.Addr{}, errors.New("no available IPs")
	}

	// We need to know the subnet broadcast address, so we don't accidentally
	// produce it.
	cidrBCastIPB := bytes.Clone(cidrIPB)
	for i := range cidrBCastIPB {
		cidrBCastIPB[i] |= ^cidrMask[i]
	}
	cidrBCastIP, ok := netip.AddrFromSlice(cidrBCastIPB)
	if !ok {
		panic(fmt.Sprintf("invalid broadcast ip calculated: %x", cidrBCastIP))
	}

	// Try a handful of times to pick an IP at random. This is preferred, as it
	// leaves less room for two different CreateHost calls to choose the same
	// IP.
	for range 20 {
		b := make([]byte, len(cidrIPB))
		if _, err := rand.Read(b); err != nil {
			return netip.Addr{}, fmt.Errorf("reading random bytes: %w", err)
		}

		for i := range b {
			b[i] = cidrIPB[i] | (b[i] & ^cidrMask[i])
		}

		ip, ok := netip.AddrFromSlice(b)
		if !ok {
			panic(fmt.Sprintf("generated invalid IP: %x", b))
		} else if !cidr.Contains(ip) {
			panic(fmt.Sprintf(
				"generated IP %v which is not in cidr %v", ip, cidr,
			))
		}

		if ip == cidrIP || ip == cidrBCastIP {
			continue
		}

		if _, inUse := inUseIPs[ip]; !inUse {
			return ip, nil
		}
	}

	// If randomly picking fails then just go through IPs one by one until the
	// free one is found.
	for ip := cidrIP.Next(); ip != cidrBCastIP; ip = ip.Next() {
		if _, inUse := inUseIPs[ip]; !inUse {
			return ip, nil
		}
	}

	panic("All ips are in-use, but somehow that wasn't determined earlier")
}

func (d *daemon) CreateHost(
	ctx context.Context,
	hostName nebula.HostName,
	opts CreateHostOpts,
) (
	JoiningBootstrap, error,
) {
	d.l.RLock()
	currBootstrap := d.currBootstrap
	d.l.RUnlock()

	ip := opts.IP
	if ip == (netip.Addr{}) {
		var err error
		if ip, err = chooseAvailableIP(currBootstrap); err != nil {
			return JoiningBootstrap{}, fmt.Errorf(
				"choosing available IP: %w", err,
			)
		}
	}

	caSigningPrivateKey, err := getNebulaCASigningPrivateKey(
		ctx, d.secretsStore,
	)
	if err != nil {
		return JoiningBootstrap{}, fmt.Errorf("getting CA signing key: %w", err)
	}

	var joiningBootstrap JoiningBootstrap
	joiningBootstrap.Bootstrap, err = bootstrap.New(
		makeCACreds(currBootstrap, caSigningPrivateKey),
		currBootstrap.NetworkCreationParams,
		currBootstrap.Hosts,
		hostName,
		ip,
	)
	if err != nil {
		return JoiningBootstrap{}, fmt.Errorf(
			"initializing bootstrap data: %w", err,
		)
	}

	secretsIDs := []secrets.ID{
		garageRPCSecretSecretID,
		garageS3APIGlobalBucketCredentialsSecretID,
	}

	if opts.CanCreateHosts {
		secretsIDs = append(secretsIDs, nebulaCASigningPrivateKeySecretID)
	}

	if joiningBootstrap.Secrets, err = secrets.Export(
		ctx, d.secretsStore, secretsIDs,
	); err != nil {
		return JoiningBootstrap{}, fmt.Errorf("exporting secrets: %w", err)
	}

	d.logger.Info(ctx, "Putting new host in garage")
	err = d.putGarageBoostrapHost(ctx, joiningBootstrap.Bootstrap)
	if err != nil {
		return JoiningBootstrap{}, fmt.Errorf("putting new host in garage: %w", err)
	}

	// the new bootstrap will have been initialized with both all existing hosts
	// (based on currBootstrap) and the host being created.
	newHosts := joiningBootstrap.Bootstrap.Hosts

	d.logger.Info(ctx, "Reloading local state with new host")
	if err := d.reload(ctx, currBootstrap, newHosts); err != nil {
		return JoiningBootstrap{}, fmt.Errorf("reloading child processes: %w", err)
	}

	return joiningBootstrap, nil
}

func (d *daemon) CreateNebulaCertificate(
	ctx context.Context,
	hostName nebula.HostName,
	hostPubKey nebula.EncryptingPublicKey,
	opts CreateNebulaCertificateOpts,
) (
	nebula.Certificate, error,
) {
	return withCurrBootstrap(d, func(
		currBootstrap bootstrap.Bootstrap,
	) (
		nebula.Certificate, error,
	) {
		ip := opts.IP
		if ip == (netip.Addr{}) {
			host, ok := currBootstrap.Hosts[hostName]
			if !ok {
				return nebula.Certificate{}, ErrHostNotFound
			}
			ip = host.IP()
		}

		caSigningPrivateKey, err := getNebulaCASigningPrivateKey(
			ctx, d.secretsStore,
		)
		if err != nil {
			return nebula.Certificate{}, fmt.Errorf("getting CA signing key: %w", err)
		}

		caCreds := makeCACreds(currBootstrap, caSigningPrivateKey)

		return nebula.NewHostCert(caCreds, hostPubKey, hostName, ip)
	})
}

func (d *daemon) Shutdown() error {
	d.l.Lock()
	defer d.l.Unlock()

	close(d.shutdownCh)
	d.wg.Wait()
	d.state = daemonStateShutdown

	if d.children != nil {
		d.children.Shutdown()
	}

	return nil
}