isle/go/daemon/network/network_it_util_test.go

package network

import (
	"context"
	"fmt"
	"isle/bootstrap"
	"isle/daemon/children"
	"isle/daemon/daecommon"
	"isle/garage"
	"isle/nebula"
	"isle/toolkit"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"testing"

	"dev.mediocregopher.com/mediocre-go-lib.git/mlog"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// Utilities related to running network integration tests

var (
	getEnvBinDirPath = sync.OnceValue(func() string {
		appDirPath := os.Getenv("APPDIR")
		if appDirPath == "" {
			panic("APPDIR not set")
		}
		return filepath.Join(appDirPath, "usr/libexec/isle")
	})

	ipNetCounter          = new(atomic.Uint64)
	publicAddrPortCounter = func() *atomic.Uint64 {
		i := new(atomic.Uint64)
		i.Store(1024)
		return i
	}()
)

func newIPNet() nebula.IPNet {
	var (
		ipNet    nebula.IPNet
		ipNetStr = fmt.Sprintf(
			"172.16.%d.0/24", ipNetCounter.Add(1)-1,
		)
	)

	if err := ipNet.UnmarshalText([]byte(ipNetStr)); err != nil {
		panic(fmt.Sprintf("parsing IPNet from %q: %v", ipNetStr, err))
	}

	return ipNet
}

func newPublicAddr() string {
	return fmt.Sprintf(
		"127.0.0.200:%d", publicAddrPortCounter.Add(1)-1,
	)
}

type integrationHarness struct {
	ctx               context.Context
	logger            *mlog.Logger
	constructors      constructors
	rootDir           toolkit.Dir
	dirCounter        atomic.Uint64
	nebulaDeviceNamer *children.NebulaDeviceNamer
}

func newIntegrationHarness(t *testing.T) *integrationHarness {
	t.Parallel()
	toolkit.MarkIntegrationTest(t)

	rootDir, err := os.MkdirTemp("", "isle-network-it-test.*")
	require.NoError(t, err)

	t.Logf("Temporary test directory: %q", rootDir)

	t.Cleanup(func() {
		if t.Failed() {
			t.Logf("Temp directory for failed test not deleted: %q", rootDir)
			return
		}

		t.Logf("Deleting temp directory %q", rootDir)
		assert.NoError(t, os.RemoveAll(rootDir))
	})

	return &integrationHarness{
		ctx:               context.Background(),
		logger:            toolkit.NewTestLogger(t),
		constructors:      newConstructors(),
		rootDir:           toolkit.Dir{Path: rootDir},
		nebulaDeviceNamer: children.NewNebulaDeviceNamer(),
	}
}

func (h *integrationHarness) mkDir(t *testing.T, name string) toolkit.Dir {
	fullName := fmt.Sprintf("%s-%d", name, h.dirCounter.Add(1)-1)

	t.Logf("Creating directory %q", fullName)
	d, err := h.rootDir.MkChildDir(fullName, false)
	require.NoError(t, err)

	return d
}

type networkConfigOpts struct {
	hasPublicAddr    bool
	numStorageAllocs int
}

func (o *networkConfigOpts) withDefaults() *networkConfigOpts {
	if o == nil {
		o = new(networkConfigOpts)
	}

	return o
}

func (h *integrationHarness) mkNetworkConfig(
	t *testing.T,
	opts *networkConfigOpts,
) daecommon.NetworkConfig {
	if opts == nil {
		opts = new(networkConfigOpts)
	}

	return daecommon.NewNetworkConfig(func(c *daecommon.NetworkConfig) {
		if opts.hasPublicAddr {
			c.VPN.PublicAddr = newPublicAddr()
		}

		c.Storage.Allocations = make(
			[]daecommon.ConfigStorageAllocation, opts.numStorageAllocs,
		)
		for i := range c.Storage.Allocations {
			c.Storage.Allocations[i] = daecommon.ConfigStorageAllocation{
				DataPath: h.mkDir(t, "data").Path,
				MetaPath: h.mkDir(t, "meta").Path,
				Capacity: 1,
			}
		}
	})
}

type createNetworkOpts struct {
	creationParams   bootstrap.CreationParams
	manualShutdown   bool
	numStorageAllocs int
}

func (o *createNetworkOpts) withDefaults() *createNetworkOpts {
	if o == nil {
		o = new(createNetworkOpts)
	}

	if o.creationParams == (bootstrap.CreationParams{}) {
		o.creationParams = bootstrap.NewCreationParams("test", "test.localnet")
	}

	if o.numStorageAllocs == 0 {
		o.numStorageAllocs = 3
	}

	return o
}

type integrationHarnessNetwork struct {
	Network

	ctx                  context.Context
	logger               *mlog.Logger
	constructors         constructors
	hostName             nebula.HostName
	stateDir, runtimeDir toolkit.Dir
	nebulaDeviceNamer    *children.NebulaDeviceNamer
	opts                 *Opts
}

func (h *integrationHarness) createNetwork(
	t *testing.T,
	hostNameStr string,
	opts *createNetworkOpts,
) *integrationHarnessNetwork {
	t.Logf("Creating as %q", hostNameStr)
	opts = opts.withDefaults()

	var (
		logger        = h.logger.WithNamespace("network").WithNamespace(hostNameStr)
		networkConfig = h.mkNetworkConfig(t, &networkConfigOpts{
			hasPublicAddr:    true,
			numStorageAllocs: opts.numStorageAllocs,
		})

		stateDir   = h.mkDir(t, "state")
		runtimeDir = h.mkDir(t, "runtime")

		ipNet    = newIPNet()
		hostName = nebula.HostName(hostNameStr)

		networkOpts = &Opts{
			GarageAdminToken: "admin_token",
			Config:           &networkConfig,
		}
	)

	network, err := h.constructors.create(
		h.ctx,
		logger,
		getEnvBinDirPath(),
		h.nebulaDeviceNamer,
		stateDir,
		runtimeDir,
		opts.creationParams,
		ipNet,
		hostName,
		networkOpts,
	)
	if err != nil {
		t.Fatalf("creating Network: %v", err)
	}

	nh := &integrationHarnessNetwork{
		network,
		h.ctx,
		logger,
		h.constructors,
		hostName,
		stateDir,
		runtimeDir,
		h.nebulaDeviceNamer,
		networkOpts,
	}

	if !opts.manualShutdown {
		t.Cleanup(func() {
			t.Logf("Shutting down Network %q", hostNameStr)
			if err := nh.Shutdown(); err != nil {
				t.Logf("Shutting down Network %q failed: %v", hostNameStr, err)
			}
		})
	}

	return nh
}

type joinNetworkOpts struct {
	*networkConfigOpts
	canCreateHosts bool
	manualShutdown bool
}

func (o *joinNetworkOpts) withDefaults() *joinNetworkOpts {
	if o == nil {
		o = new(joinNetworkOpts)
	}

	o.networkConfigOpts = o.networkConfigOpts.withDefaults()

	return o
}

func (h *integrationHarness) joinNetwork(
	t *testing.T,
	network *integrationHarnessNetwork,
	hostNameStr string,
	opts *joinNetworkOpts,
) *integrationHarnessNetwork {
	opts = opts.withDefaults()
	hostName := nebula.HostName(hostNameStr)

	t.Logf("Creating bootstrap for %q", hostNameStr)
	joiningBootstrap, err := network.CreateHost(h.ctx, hostName, CreateHostOpts{
		CanCreateHosts: opts.canCreateHosts,
	})
	if err != nil {
		t.Fatalf("creating host joining bootstrap: %v", err)
	}

	var (
		logger        = h.logger.WithNamespace("network").WithNamespace(hostNameStr)
		networkConfig = h.mkNetworkConfig(t, opts.networkConfigOpts)
		stateDir      = h.mkDir(t, "state")
		runtimeDir    = h.mkDir(t, "runtime")
		networkOpts   = &Opts{
			GarageAdminToken: "admin_token",
			Config:           &networkConfig,
		}
	)

	t.Logf("Joining as %q", hostNameStr)
	joinedNetwork, err := h.constructors.join(
		h.ctx,
		logger,
		getEnvBinDirPath(),
		h.nebulaDeviceNamer,
		joiningBootstrap,
		stateDir,
		runtimeDir,
		networkOpts,
	)
	if err != nil {
		t.Fatalf("joining network: %v", err)
	}

	nh := &integrationHarnessNetwork{
		joinedNetwork,
		h.ctx,
		logger,
		h.constructors,
		hostName,
		stateDir,
		runtimeDir,
		h.nebulaDeviceNamer,
		networkOpts,
	}

	if !opts.manualShutdown {
		t.Cleanup(func() {
			t.Logf("Shutting down Network %q", hostNameStr)
			if err := nh.Shutdown(); err != nil {
				t.Logf("Shutting down Network %q failed: %v", hostNameStr, err)
			}
		})
	}

	return nh
}

func (nh *integrationHarnessNetwork) restart(t *testing.T) {
	t.Log("Shutting down network (restart)")
	require.NoError(t, nh.Network.Shutdown())

	t.Log("Loading network (restart)")
	var err error
	nh.Network, err = nh.constructors.load(
		nh.ctx,
		nh.logger,
		getEnvBinDirPath(),
		nh.nebulaDeviceNamer,
		nh.stateDir,
		nh.runtimeDir,
		nh.opts,
	)
	require.NoError(t, err)
}

func (nh *integrationHarnessNetwork) getConfig(t *testing.T) daecommon.NetworkConfig {
	networkConfig, err := nh.Network.GetConfig(nh.ctx)
	require.NoError(t, err)
	return networkConfig
}

func (nh *integrationHarnessNetwork) getBootstrap(
	t *testing.T,
) bootstrap.Bootstrap {
	currBootstrap, err := nh.Network.(*network).getBootstrap()
	require.NoError(t, err)
	return currBootstrap
}

func (nh *integrationHarnessNetwork) garageAdminClient(
	t *testing.T,
) *garage.AdminClient {
	c := newGarageAdminClient(
		nh.logger,
		nh.getConfig(t),
		nh.opts.GarageAdminToken,
		nh.getBootstrap(t).ThisHost(),
	)
	t.Cleanup(func() { assert.NoError(t, c.Close()) })
	return c
}

func (nh *integrationHarnessNetwork) getHostsByName(
	t *testing.T,
) map[nebula.HostName]bootstrap.Host {
	currBootstrap, err := nh.Network.GetBootstrap(nh.ctx)
	require.NoError(t, err)
	return currBootstrap.Hosts
}