package network import ( "context" "fmt" "isle/bootstrap" "isle/daemon/children" "isle/daemon/daecommon" "isle/garage" "isle/nebula" "isle/toolkit" "os" "path/filepath" "sync" "sync/atomic" "testing" "dev.mediocregopher.com/mediocre-go-lib.git/mlog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // Utilities related to running network integration tests var ( getEnvBinDirPath = sync.OnceValue(func() string { appDirPath := os.Getenv("APPDIR") if appDirPath == "" { panic("APPDIR not set") } return filepath.Join(appDirPath, "usr/libexec/isle") }) ipNetCounter = new(atomic.Uint64) publicAddrPortCounter = func() *atomic.Uint64 { i := new(atomic.Uint64) i.Store(1024) return i }() ) func newIPNet() nebula.IPNet { var ( ipNet nebula.IPNet ipNetStr = fmt.Sprintf( "172.16.%d.0/24", ipNetCounter.Add(1)-1, ) ) if err := ipNet.UnmarshalText([]byte(ipNetStr)); err != nil { panic(fmt.Sprintf("parsing IPNet from %q: %v", ipNetStr, err)) } return ipNet } func newPublicAddr() string { return fmt.Sprintf( "127.0.0.200:%d", publicAddrPortCounter.Add(1)-1, ) } type integrationHarness struct { ctx context.Context logger *mlog.Logger constructors constructors rootDir toolkit.Dir dirCounter atomic.Uint64 nebulaDeviceNamer *children.NebulaDeviceNamer } func newIntegrationHarness(t *testing.T) *integrationHarness { t.Parallel() toolkit.MarkIntegrationTest(t) rootDir, err := os.MkdirTemp("", "isle-network-it-test.*") require.NoError(t, err) t.Logf("Temporary test directory: %q", rootDir) t.Cleanup(func() { if t.Failed() { t.Logf("Temp directory for failed test not deleted: %q", rootDir) return } t.Logf("Deleting temp directory %q", rootDir) assert.NoError(t, os.RemoveAll(rootDir)) }) return &integrationHarness{ ctx: context.Background(), logger: toolkit.NewTestLogger(t), constructors: newConstructors(), rootDir: toolkit.Dir{Path: rootDir}, nebulaDeviceNamer: children.NewNebulaDeviceNamer(), } } func (h *integrationHarness) mkDir(t *testing.T, name string) toolkit.Dir { fullName := fmt.Sprintf("%s-%d", name, h.dirCounter.Add(1)-1) t.Logf("Creating directory %q", fullName) d, err := h.rootDir.MkChildDir(fullName, false) require.NoError(t, err) return d } type networkConfigOpts struct { hasPublicAddr bool numStorageAllocs int } func (o *networkConfigOpts) withDefaults() *networkConfigOpts { if o == nil { o = new(networkConfigOpts) } return o } func (h *integrationHarness) mkNetworkConfig( t *testing.T, opts *networkConfigOpts, ) daecommon.NetworkConfig { if opts == nil { opts = new(networkConfigOpts) } return daecommon.NewNetworkConfig(func(c *daecommon.NetworkConfig) { if opts.hasPublicAddr { c.VPN.PublicAddr = newPublicAddr() } c.Storage.Allocations = make( []daecommon.ConfigStorageAllocation, opts.numStorageAllocs, ) for i := range c.Storage.Allocations { c.Storage.Allocations[i] = daecommon.ConfigStorageAllocation{ DataPath: h.mkDir(t, "data").Path, MetaPath: h.mkDir(t, "meta").Path, Capacity: 1, } } }) } type createNetworkOpts struct { creationParams bootstrap.CreationParams manualShutdown bool numStorageAllocs int } func (o *createNetworkOpts) withDefaults() *createNetworkOpts { if o == nil { o = new(createNetworkOpts) } if o.creationParams == (bootstrap.CreationParams{}) { o.creationParams = bootstrap.NewCreationParams("test", "test.localnet") } if o.numStorageAllocs == 0 { o.numStorageAllocs = 3 } return o } type integrationHarnessNetwork struct { Network ctx context.Context logger *mlog.Logger constructors constructors hostName nebula.HostName stateDir, runtimeDir toolkit.Dir nebulaDeviceNamer *children.NebulaDeviceNamer opts *Opts } func (h *integrationHarness) createNetwork( t *testing.T, hostNameStr string, opts *createNetworkOpts, ) *integrationHarnessNetwork { t.Logf("Creating as %q", hostNameStr) opts = opts.withDefaults() var ( logger = h.logger.WithNamespace("network").WithNamespace(hostNameStr) networkConfig = h.mkNetworkConfig(t, &networkConfigOpts{ hasPublicAddr: true, numStorageAllocs: opts.numStorageAllocs, }) stateDir = h.mkDir(t, "state") runtimeDir = h.mkDir(t, "runtime") ipNet = newIPNet() hostName = nebula.HostName(hostNameStr) networkOpts = &Opts{ GarageAdminToken: "admin_token", Config: &networkConfig, } ) network, err := h.constructors.create( h.ctx, logger, getEnvBinDirPath(), h.nebulaDeviceNamer, stateDir, runtimeDir, opts.creationParams, ipNet, hostName, networkOpts, ) if err != nil { t.Fatalf("creating Network: %v", err) } nh := &integrationHarnessNetwork{ network, h.ctx, logger, h.constructors, hostName, stateDir, runtimeDir, h.nebulaDeviceNamer, networkOpts, } if !opts.manualShutdown { t.Cleanup(func() { t.Logf("Shutting down Network %q", hostNameStr) if err := nh.Shutdown(); err != nil { t.Logf("Shutting down Network %q failed: %v", hostNameStr, err) } }) } return nh } type joinNetworkOpts struct { *networkConfigOpts canCreateHosts bool manualShutdown bool } func (o *joinNetworkOpts) withDefaults() *joinNetworkOpts { if o == nil { o = new(joinNetworkOpts) } o.networkConfigOpts = o.networkConfigOpts.withDefaults() return o } func (h *integrationHarness) joinNetwork( t *testing.T, network *integrationHarnessNetwork, hostNameStr string, opts *joinNetworkOpts, ) *integrationHarnessNetwork { opts = opts.withDefaults() hostName := nebula.HostName(hostNameStr) t.Logf("Creating bootstrap for %q", hostNameStr) joiningBootstrap, err := network.CreateHost(h.ctx, hostName, CreateHostOpts{ CanCreateHosts: opts.canCreateHosts, }) if err != nil { t.Fatalf("creating host joining bootstrap: %v", err) } var ( logger = h.logger.WithNamespace("network").WithNamespace(hostNameStr) networkConfig = h.mkNetworkConfig(t, opts.networkConfigOpts) stateDir = h.mkDir(t, "state") runtimeDir = h.mkDir(t, "runtime") networkOpts = &Opts{ GarageAdminToken: "admin_token", Config: &networkConfig, } ) t.Logf("Joining as %q", hostNameStr) joinedNetwork, err := h.constructors.join( h.ctx, logger, getEnvBinDirPath(), h.nebulaDeviceNamer, joiningBootstrap, stateDir, runtimeDir, networkOpts, ) if err != nil { t.Fatalf("joining network: %v", err) } nh := &integrationHarnessNetwork{ joinedNetwork, h.ctx, logger, h.constructors, hostName, stateDir, runtimeDir, h.nebulaDeviceNamer, networkOpts, } if !opts.manualShutdown { t.Cleanup(func() { t.Logf("Shutting down Network %q", hostNameStr) if err := nh.Shutdown(); err != nil { t.Logf("Shutting down Network %q failed: %v", hostNameStr, err) } }) } return nh } func (nh *integrationHarnessNetwork) restart(t *testing.T) { t.Log("Shutting down network (restart)") require.NoError(t, nh.Network.Shutdown()) t.Log("Loading network (restart)") var err error nh.Network, err = nh.constructors.load( nh.ctx, nh.logger, getEnvBinDirPath(), nh.nebulaDeviceNamer, nh.stateDir, nh.runtimeDir, nh.opts, ) require.NoError(t, err) } func (nh *integrationHarnessNetwork) getConfig(t *testing.T) daecommon.NetworkConfig { networkConfig, err := nh.Network.GetConfig(nh.ctx) require.NoError(t, err) return networkConfig } func (nh *integrationHarnessNetwork) getBootstrap( t *testing.T, ) bootstrap.Bootstrap { currBootstrap, err := nh.Network.(*network).getBootstrap() require.NoError(t, err) return currBootstrap } func (nh *integrationHarnessNetwork) garageAdminClient( t *testing.T, ) *garage.AdminClient { c := newGarageAdminClient( nh.logger, nh.getConfig(t), nh.opts.GarageAdminToken, nh.getBootstrap(t).ThisHost(), ) t.Cleanup(func() { assert.NoError(t, c.Close()) }) return c } func (nh *integrationHarnessNetwork) getHostsByName( t *testing.T, ) map[nebula.HostName]bootstrap.Host { currBootstrap, err := nh.Network.GetBootstrap(nh.ctx) require.NoError(t, err) return currBootstrap.Hosts }