// Package network implements the Network type, which manages the daemon's // membership in a single network. package network import ( "bytes" "cmp" "context" "crypto/rand" "encoding/json" "errors" "fmt" "isle/bootstrap" "isle/daemon/children" "isle/daemon/daecommon" "isle/garage" "isle/jsonutil" "isle/nebula" "isle/secrets" "isle/toolkit" "log" "net/netip" "slices" "sync" "time" "dev.mediocregopher.com/mediocre-go-lib.git/mlog" "golang.org/x/exp/maps" ) // GarageClientParams contains all the data needed to instantiate garage // clients. type GarageClientParams struct { Peer garage.RemotePeer GlobalBucketS3APICredentials garage.S3APICredentials // RPCSecret may be empty, if the secret is not available on the host. RPCSecret string } // GlobalBucketS3APIClient returns an S3 client pre-configured with access to // the global bucket. func (p GarageClientParams) GlobalBucketS3APIClient() garage.S3APIClient { var ( addr = p.Peer.S3APIAddr() creds = p.GlobalBucketS3APICredentials ) return garage.NewS3APIClient(addr, creds) } // CreateHostOpts are optional parameters to the CreateHost method. type CreateHostOpts struct { // IP address of the new host. An IP address will be randomly chosen if one // is not given here. IP netip.Addr // CanCreateHosts indicates that the bootstrap produced by CreateHost should // give the new host the ability to create new hosts as well. CanCreateHosts bool // TODO add nebula cert tags } // JoiningBootstrap wraps a normal Bootstrap to include extra data which a host // might need while joining a network. type JoiningBootstrap struct { Bootstrap bootstrap.Bootstrap Secrets map[secrets.ID]json.RawMessage } // RPC defines the methods related to a single network which are available over // the daemon's RPC interface. type RPC interface { // GetHosts returns all hosts known to the network, sorted by their name. GetHosts(context.Context) ([]bootstrap.Host, error) // GetGarageClientParams returns a GarageClientParams for the current // network state. GetGarageClientParams(context.Context) (GarageClientParams, error) // GetNebulaCAPublicCredentials returns the CAPublicCredentials for the // network. GetNebulaCAPublicCredentials( context.Context, ) ( nebula.CAPublicCredentials, error, ) // RemoveHost removes the host of the given name from the network. RemoveHost(ctx context.Context, hostName nebula.HostName) error // CreateHost creates a bootstrap for a new host with the given name and IP // address. CreateHost( context.Context, nebula.HostName, CreateHostOpts, ) ( JoiningBootstrap, error, ) // CreateNebulaCertificate creates and signs a new nebula certficate for an // existing host, given the public key for that host. This is currently // mostly useful for creating certs for mobile devices. // // TODO replace this with CreateHostBootstrap, and the // CreateNebulaCertificate RPC method can just pull cert out of that. // // Errors: // - ErrHostNotFound CreateNebulaCertificate( context.Context, nebula.HostName, nebula.EncryptingPublicKey, ) ( nebula.Certificate, error, ) } // Network manages membership in a single micropelago network. Each Network // is comprised of a unique IP subnet, hosts connected together on that subnet // via a VPN, an S3 storage layer only accessible to those hosts, plus other // services built on this foundation. // // A single daemon (isle server) can manage multiple networks. Each network is // expected to be independent of the others, ie they should not share any // resources. type Network interface { RPC // GetNetworkCreationParams returns the CreationParams that the Network was // originally created with. GetNetworkCreationParams(context.Context) (bootstrap.CreationParams, error) // Shutdown blocks until all resources held or created by the Network, // including child processes it has started, have been cleaned up. // // If this returns an error then it's possible that child processes are // still running and are no longer managed. Shutdown() error } //////////////////////////////////////////////////////////////////////////////// // Network implementation // Opts are optional parameters which can be passed in when initializing a new // Network instance. A nil Opts is equivalent to a zero value. type Opts struct { ChildrenOpts *children.Opts } func (o *Opts) withDefaults() *Opts { if o == nil { o = new(Opts) } return o } type network struct { logger *mlog.Logger daemonConfig daecommon.Config envBinDirPath string stateDir toolkit.Dir runtimeDir toolkit.Dir opts *Opts secretsStore secrets.Store garageAdminToken string l sync.RWMutex children *children.Children currBootstrap bootstrap.Bootstrap shutdownCh chan struct{} wg sync.WaitGroup } // instatiateNetwork returns an instantiated *network instance which has not yet // been initialized. func instatiateNetwork( logger *mlog.Logger, networkID string, daemonConfig daecommon.Config, envBinDirPath string, stateDir toolkit.Dir, runtimeDir toolkit.Dir, opts *Opts, ) *network { log.Printf("DEBUG: network stateDir:%+v runtimeDir:%+v", stateDir, runtimeDir) return &network{ logger: logger, daemonConfig: daemonConfig, envBinDirPath: envBinDirPath, stateDir: stateDir, runtimeDir: runtimeDir, opts: opts.withDefaults(), garageAdminToken: randStr(32), shutdownCh: make(chan struct{}), } } // LoadCreationParams returns the CreationParams of a Network which was // Created/Joined with the given state directory. func LoadCreationParams( stateDir toolkit.Dir, ) ( bootstrap.CreationParams, error, ) { var ( // TODO store/load the creation params separately from the rest of // the bootstrap, since the bootstrap contains potentially the // entire host list of a network, which could be pretty bulky. bootstrapFilePath = bootstrap.StateDirPath(stateDir.Path) bs bootstrap.Bootstrap ) if err := jsonutil.LoadFile(&bs, bootstrapFilePath); err != nil { return bootstrap.CreationParams{}, fmt.Errorf( "loading bootstrap from %q: %w", bootstrapFilePath, err, ) } return bs.NetworkCreationParams, nil } // Load initializes and returns a Network instance for a network which was // previously joined or created, and which has the given ID. func Load( ctx context.Context, logger *mlog.Logger, networkID string, daemonConfig daecommon.Config, envBinDirPath string, stateDir toolkit.Dir, runtimeDir toolkit.Dir, opts *Opts, ) ( Network, error, ) { n := instatiateNetwork( logger, networkID, daemonConfig, envBinDirPath, stateDir, runtimeDir, opts, ) if err := n.initializeDirs(true); err != nil { return nil, fmt.Errorf("initializing directories: %w", err) } var ( currBootstrap bootstrap.Bootstrap bootstrapFilePath = bootstrap.StateDirPath(n.stateDir.Path) ) if err := jsonutil.LoadFile(&currBootstrap, bootstrapFilePath); err != nil { return nil, fmt.Errorf( "loading bootstrap from %q: %w", bootstrapFilePath, err, ) } else if err := n.initialize(ctx, currBootstrap); err != nil { return nil, fmt.Errorf("initializing with bootstrap: %w", err) } return n, nil } // Join initializes and returns a Network instance for an existing network which // was not previously joined to on this host. Once Join has been called for a // particular network it will error on subsequent calls for that same network, // Load should be used instead. func Join( ctx context.Context, logger *mlog.Logger, daemonConfig daecommon.Config, joiningBootstrap JoiningBootstrap, envBinDirPath string, stateDir toolkit.Dir, runtimeDir toolkit.Dir, opts *Opts, ) ( Network, error, ) { n := instatiateNetwork( logger, joiningBootstrap.Bootstrap.NetworkCreationParams.ID, daemonConfig, envBinDirPath, stateDir, runtimeDir, opts, ) if err := n.initializeDirs(false); err != nil { return nil, fmt.Errorf("initializing directories: %w", err) } if err := secrets.Import( ctx, n.secretsStore, joiningBootstrap.Secrets, ); err != nil { return nil, fmt.Errorf("importing secrets: %w", err) } if err := n.initialize(ctx, joiningBootstrap.Bootstrap); err != nil { return nil, fmt.Errorf("initializing with bootstrap: %w", err) } return n, nil } // Create initializes and returns a Network for a brand new network which uses // the given creation parameters. // // - name: Human-readable name of the network. // - domain: Primary domain name that network services are served under. // - ipNet: An IP subnet, in CIDR form, which will be the overall range of // possible IPs in the network. The first IP in this network range will // become this first host's IP. // - hostName: The name of this first host in the network. // // Errors: // - ErrInvalidConfig - if daemonConfig doesn't have 3 storage allocations // configured. func Create( ctx context.Context, logger *mlog.Logger, daemonConfig daecommon.Config, envBinDirPath string, stateDir toolkit.Dir, runtimeDir toolkit.Dir, creationParams bootstrap.CreationParams, ipNet nebula.IPNet, // TODO should this go in CreationParams? hostName nebula.HostName, opts *Opts, ) ( Network, error, ) { if len(daemonConfig.Storage.Allocations) < 3 { return nil, ErrInvalidConfig.WithData( "At least three storage allocations are required.", ) } nebulaCACreds, err := nebula.NewCACredentials(creationParams.Domain, ipNet) if err != nil { return nil, fmt.Errorf("creating nebula CA cert: %w", err) } garageRPCSecret := randStr(32) n := instatiateNetwork( logger, creationParams.ID, daemonConfig, envBinDirPath, stateDir, runtimeDir, opts, ) if err := n.initializeDirs(false); err != nil { return nil, fmt.Errorf("initializing directories: %w", err) } err = daecommon.SetGarageRPCSecret(ctx, n.secretsStore, garageRPCSecret) if err != nil { return nil, fmt.Errorf("setting garage RPC secret: %w", err) } err = daecommon.SetNebulaCASigningPrivateKey( ctx, n.secretsStore, nebulaCACreds.SigningPrivateKey, ) if err != nil { return nil, fmt.Errorf("setting nebula CA signing key secret: %w", err) } hostBootstrap, err := bootstrap.New( nebulaCACreds, creationParams, map[nebula.HostName]bootstrap.Host{}, hostName, ipNet.FirstAddr(), ) if err != nil { return nil, fmt.Errorf("initializing bootstrap data: %w", err) } if err := n.initialize(ctx, hostBootstrap); err != nil { return nil, fmt.Errorf("initializing with bootstrap: %w", err) } return n, nil } func (n *network) initializeDirs(mayExist bool) error { secretsDir, err := n.stateDir.MkChildDir("secrets", mayExist) if err != nil { return fmt.Errorf("creating secrets dir: %w", err) } n.secretsStore = secrets.NewFSStore(secretsDir.Path) return nil } func (n *network) initialize( ctx context.Context, currBootstrap bootstrap.Bootstrap, ) error { // we update this Host's data using whatever configuration has been provided // by the daemon config. This way the network has the most up-to-date // possible bootstrap. This updated bootstrap will later get updated in // garage as a background task, so other hosts will see it as well. currBootstrap, err := coalesceDaemonConfigAndBootstrap( n.daemonConfig, currBootstrap, ) if err != nil { return fmt.Errorf("combining configuration into bootstrap: %w", err) } err = writeBootstrapToStateDir(n.stateDir.Path, currBootstrap) if err != nil { return fmt.Errorf("writing bootstrap to state dir: %w", err) } n.currBootstrap = currBootstrap n.logger.Info(ctx, "Creating child processes") n.children, err = children.New( ctx, n.logger.WithNamespace("children"), n.envBinDirPath, n.secretsStore, n.daemonConfig, n.runtimeDir, n.garageAdminToken, currBootstrap, n.opts.ChildrenOpts, ) if err != nil { return fmt.Errorf("creating child processes: %w", err) } n.logger.Info(ctx, "Child processes created") if err := n.postInit(ctx); err != nil { n.logger.Error(ctx, "Post-initialization failed, stopping child processes", err) n.children.Shutdown() return fmt.Errorf("performing post-initialization: %w", err) } // TODO annotate this context with creation params ctx, cancel := context.WithCancel(context.Background()) n.wg.Add(1) go func() { defer n.wg.Done() <-n.shutdownCh cancel() }() n.wg.Add(1) go func() { defer n.wg.Done() n.reloadLoop(ctx) n.logger.Debug(ctx, "Daemon restart loop stopped") }() return nil } func (n *network) postInit(ctx context.Context) error { if len(n.daemonConfig.Storage.Allocations) > 0 { n.logger.Info(ctx, "Applying garage layout") if err := garageApplyLayout( ctx, n.logger, n.daemonConfig, n.garageAdminToken, n.currBootstrap, ); err != nil { return fmt.Errorf("applying garage layout: %w", err) } } // This is only necessary during network creation, otherwise the bootstrap // should already have these credentials built in. // // TODO this is pretty hacky, but there doesn't seem to be a better way to // manage it at the moment. _, err := daecommon.GetGarageS3APIGlobalBucketCredentials( ctx, n.secretsStore, ) if errors.Is(err, secrets.ErrNotFound) { n.logger.Info(ctx, "Initializing garage shared global bucket") garageGlobalBucketCreds, err := garageInitializeGlobalBucket( ctx, n.logger, n.daemonConfig, n.garageAdminToken, n.currBootstrap, ) if err != nil { return fmt.Errorf("initializing global bucket: %w", err) } err = daecommon.SetGarageS3APIGlobalBucketCredentials( ctx, n.secretsStore, garageGlobalBucketCreds, ) if err != nil { return fmt.Errorf("storing global bucket creds: %w", err) } } n.logger.Info(ctx, "Updating host info in garage") err = n.putGarageBoostrapHost(ctx, n.currBootstrap) if err != nil { return fmt.Errorf("updating host info in garage: %w", err) } return nil } func (n *network) reloadLoop(ctx context.Context) { ticker := time.NewTicker(3 * time.Minute) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: n.l.RLock() currBootstrap := n.currBootstrap n.l.RUnlock() n.logger.Info(ctx, "Checking for bootstrap changes") newHosts, err := n.getGarageBootstrapHosts(ctx, currBootstrap) if err != nil { n.logger.Error(ctx, "Failed to get hosts from garage", err) continue } // TODO there's some potential race conditions here, where // CreateHost could be called at this point, write the new host to // garage and the bootstrap, but then this reload call removes the // host from this bootstrap/children until the next reload. if err := n.reload(ctx, currBootstrap, newHosts); err != nil { n.logger.Error(ctx, "Reloading with new host data failed", err) continue } } } } // reload will check the existing hosts data from currBootstrap against a // potentially updated set of hosts data, and if there are any differences will // perform whatever changes are necessary. func (n *network) reload( ctx context.Context, currBootstrap bootstrap.Bootstrap, newHosts map[nebula.HostName]bootstrap.Host, ) error { var ( newBootstrap = currBootstrap thisHost = currBootstrap.ThisHost() ) newBootstrap.Hosts = newHosts // the daemon's view of this host's bootstrap info takes precedence over // whatever is in garage newBootstrap.Hosts[thisHost.Name] = thisHost diff, err := children.CalculateReloadDiff( n.daemonConfig, currBootstrap, newBootstrap, ) if err != nil { return fmt.Errorf("calculating diff between bootstraps: %w", err) } else if diff == (children.ReloadDiff{}) { n.logger.Info(ctx, "No changes to bootstrap detected") return nil } n.logger.Info(ctx, "Bootstrap has changed, storing new bootstrap") n.l.Lock() n.currBootstrap = newBootstrap n.l.Unlock() if err := n.children.Reload(ctx, newBootstrap, diff); err != nil { return fmt.Errorf("reloading child processes (diff:%+v): %w", diff, err) } return nil } func withCurrBootstrap[Res any]( n *network, fn func(bootstrap.Bootstrap) (Res, error), ) (Res, error) { n.l.RLock() defer n.l.RUnlock() currBootstrap := n.currBootstrap return fn(currBootstrap) } func (n *network) getBootstrap( ctx context.Context, ) ( bootstrap.Bootstrap, error, ) { return withCurrBootstrap(n, func( currBootstrap bootstrap.Bootstrap, ) ( bootstrap.Bootstrap, error, ) { return currBootstrap, nil }) } func (n *network) GetHosts(ctx context.Context) ([]bootstrap.Host, error) { b, err := n.getBootstrap(ctx) if err != nil { return nil, fmt.Errorf("retrieving bootstrap: %w", err) } hosts := maps.Values(b.Hosts) slices.SortFunc(hosts, func(a, b bootstrap.Host) int { return cmp.Compare(a.Name, b.Name) }) return hosts, nil } func (n *network) GetGarageClientParams( ctx context.Context, ) ( GarageClientParams, error, ) { return withCurrBootstrap(n, func( currBootstrap bootstrap.Bootstrap, ) ( GarageClientParams, error, ) { return n.getGarageClientParams(ctx, currBootstrap) }) } func (n *network) GetNebulaCAPublicCredentials( ctx context.Context, ) ( nebula.CAPublicCredentials, error, ) { b, err := n.getBootstrap(ctx) if err != nil { return nebula.CAPublicCredentials{}, fmt.Errorf( "retrieving bootstrap: %w", err, ) } return b.CAPublicCredentials, nil } func (n *network) RemoveHost(ctx context.Context, hostName nebula.HostName) error { // TODO RemoveHost should publish a certificate revocation for the host // being removed. _, err := withCurrBootstrap(n, func( currBootstrap bootstrap.Bootstrap, ) ( struct{}, error, ) { garageClientParams, err := n.getGarageClientParams(ctx, currBootstrap) if err != nil { return struct{}{}, fmt.Errorf("get garage client params: %w", err) } client := garageClientParams.GlobalBucketS3APIClient() return struct{}{}, removeGarageBootstrapHost(ctx, client, hostName) }) return err } func makeCACreds( currBootstrap bootstrap.Bootstrap, caSigningPrivateKey nebula.SigningPrivateKey, ) nebula.CACredentials { return nebula.CACredentials{ Public: currBootstrap.CAPublicCredentials, SigningPrivateKey: caSigningPrivateKey, } } func chooseAvailableIP(b bootstrap.Bootstrap) (netip.Addr, error) { var ( cidrIPNet = b.CAPublicCredentials.Cert.Unwrap().Details.Subnets[0] cidrMask = cidrIPNet.Mask cidrIPB = cidrIPNet.IP cidr = netip.MustParsePrefix(cidrIPNet.String()) cidrIP = cidr.Addr() cidrSuffixBits = cidrIP.BitLen() - cidr.Bits() inUseIPs = make(map[netip.Addr]struct{}, len(b.Hosts)) ) for _, host := range b.Hosts { inUseIPs[host.IP()] = struct{}{} } // first check that there are any addresses at all. We can determine the // number of possible addresses using the network CIDR. The first IP in a // subnet is the network identifier, and is reserved. The last IP is the // broadcast IP, and is also reserved. Hence, the -2. usableIPs := (1 << cidrSuffixBits) - 2 if len(inUseIPs) >= usableIPs { return netip.Addr{}, errors.New("no available IPs") } // We need to know the subnet broadcast address, so we don't accidentally // produce it. cidrBCastIPB := bytes.Clone(cidrIPB) for i := range cidrBCastIPB { cidrBCastIPB[i] |= ^cidrMask[i] } cidrBCastIP, ok := netip.AddrFromSlice(cidrBCastIPB) if !ok { panic(fmt.Sprintf("invalid broadcast ip calculated: %x", cidrBCastIP)) } // Try a handful of times to pick an IP at random. This is preferred, as it // leaves less room for two different CreateHost calls to choose the same // IP. for range 20 { b := make([]byte, len(cidrIPB)) if _, err := rand.Read(b); err != nil { return netip.Addr{}, fmt.Errorf("reading random bytes: %w", err) } for i := range b { b[i] = cidrIPB[i] | (b[i] & ^cidrMask[i]) } ip, ok := netip.AddrFromSlice(b) if !ok { panic(fmt.Sprintf("generated invalid IP: %x", b)) } else if !cidr.Contains(ip) { panic(fmt.Sprintf( "generated IP %v which is not in cidr %v", ip, cidr, )) } if ip == cidrIP || ip == cidrBCastIP { continue } if _, inUse := inUseIPs[ip]; !inUse { return ip, nil } } // If randomly picking fails then just go through IPs one by one until the // free one is found. for ip := cidrIP.Next(); ip != cidrBCastIP; ip = ip.Next() { if _, inUse := inUseIPs[ip]; !inUse { return ip, nil } } panic("All ips are in-use, but somehow that wasn't determined earlier") } func (n *network) CreateHost( ctx context.Context, hostName nebula.HostName, opts CreateHostOpts, ) ( JoiningBootstrap, error, ) { n.l.RLock() currBootstrap := n.currBootstrap n.l.RUnlock() ip := opts.IP if ip == (netip.Addr{}) { var err error if ip, err = chooseAvailableIP(currBootstrap); err != nil { return JoiningBootstrap{}, fmt.Errorf( "choosing available IP: %w", err, ) } } // TODO if the ip is given, check that it's not already in use. caSigningPrivateKey, err := daecommon.GetNebulaCASigningPrivateKey( ctx, n.secretsStore, ) if err != nil { return JoiningBootstrap{}, fmt.Errorf("getting CA signing key: %w", err) } var joiningBootstrap JoiningBootstrap joiningBootstrap.Bootstrap, err = bootstrap.New( makeCACreds(currBootstrap, caSigningPrivateKey), currBootstrap.NetworkCreationParams, currBootstrap.Hosts, hostName, ip, ) if err != nil { return JoiningBootstrap{}, fmt.Errorf( "initializing bootstrap data: %w", err, ) } secretsIDs := []secrets.ID{ daecommon.GarageRPCSecretSecretID, daecommon.GarageS3APIGlobalBucketCredentialsSecretID, } if opts.CanCreateHosts { secretsIDs = append( secretsIDs, daecommon.NebulaCASigningPrivateKeySecretID, ) } if joiningBootstrap.Secrets, err = secrets.Export( ctx, n.secretsStore, secretsIDs, ); err != nil { return JoiningBootstrap{}, fmt.Errorf("exporting secrets: %w", err) } n.logger.Info(ctx, "Putting new host in garage") err = n.putGarageBoostrapHost(ctx, joiningBootstrap.Bootstrap) if err != nil { return JoiningBootstrap{}, fmt.Errorf("putting new host in garage: %w", err) } // the new bootstrap will have been initialized with both all existing hosts // (based on currBootstrap) and the host being created. newHosts := joiningBootstrap.Bootstrap.Hosts n.logger.Info(ctx, "Reloading local state with new host") if err := n.reload(ctx, currBootstrap, newHosts); err != nil { return JoiningBootstrap{}, fmt.Errorf("reloading child processes: %w", err) } return joiningBootstrap, nil } func (n *network) CreateNebulaCertificate( ctx context.Context, hostName nebula.HostName, hostPubKey nebula.EncryptingPublicKey, ) ( nebula.Certificate, error, ) { return withCurrBootstrap(n, func( currBootstrap bootstrap.Bootstrap, ) ( nebula.Certificate, error, ) { host, ok := currBootstrap.Hosts[hostName] if !ok { return nebula.Certificate{}, ErrHostNotFound } ip := host.IP() caSigningPrivateKey, err := daecommon.GetNebulaCASigningPrivateKey( ctx, n.secretsStore, ) if err != nil { return nebula.Certificate{}, fmt.Errorf("getting CA signing key: %w", err) } caCreds := makeCACreds(currBootstrap, caSigningPrivateKey) return nebula.NewHostCert(caCreds, hostPubKey, hostName, ip) }) } func (n *network) GetNetworkCreationParams( ctx context.Context, ) ( bootstrap.CreationParams, error, ) { return withCurrBootstrap(n, func( currBootstrap bootstrap.Bootstrap, ) ( bootstrap.CreationParams, error, ) { return currBootstrap.NetworkCreationParams, nil }) } func (n *network) Shutdown() error { close(n.shutdownCh) n.wg.Wait() if n.children != nil { n.children.Shutdown() } return nil }