package network import ( "context" "encoding/json" "fmt" "isle/bootstrap" "isle/daemon/daecommon" "isle/garage" "isle/nebula" "isle/toolkit" "os" "path/filepath" "sync" "sync/atomic" "testing" "dev.mediocregopher.com/mediocre-go-lib.git/mlog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" ) // Utilities related to running network integration tests var ( getEnvBinDirPath = sync.OnceValue(func() string { appDirPath := os.Getenv("APPDIR") if appDirPath == "" { panic("APPDIR not set") } return filepath.Join(appDirPath, "bin") }) ipNetCounter uint64 = 0 publicAddrPortCounter uint64 = 1024 tunDeviceCounter uint64 = 0 ) func newIPNet() nebula.IPNet { var ( ipNet nebula.IPNet ipNetStr = fmt.Sprintf( "172.16.%d.0/24", atomic.AddUint64(&ipNetCounter, 1), ) ) if err := ipNet.UnmarshalText([]byte(ipNetStr)); err != nil { panic(fmt.Sprintf("parsing IPNet from %q: %v", ipNetStr, err)) } return ipNet } func newPublicAddr() string { return fmt.Sprintf( "127.0.0.200:%d", atomic.AddUint64(&publicAddrPortCounter, 1), ) } func newTunDevice() string { return fmt.Sprintf("isle-test-%d", atomic.AddUint64(&tunDeviceCounter, 1)) } func mustParseNetworkConfigf(str string, args ...any) daecommon.NetworkConfig { str = fmt.Sprintf(str, args...) var networkConfig daecommon.NetworkConfig if err := yaml.Unmarshal([]byte(str), &networkConfig); err != nil { panic(fmt.Sprintf("parsing network config: %v", err)) } return networkConfig } type integrationHarness struct { ctx context.Context logger *mlog.Logger rootDir toolkit.Dir dirCounter uint64 } func newIntegrationHarness(t *testing.T) *integrationHarness { t.Parallel() toolkit.MarkIntegrationTest(t) rootDir, err := os.MkdirTemp("", "isle-network-it-test.*") require.NoError(t, err) t.Logf("Temporary test directory: %q", rootDir) t.Cleanup(func() { if t.Failed() { t.Logf("Temp directory for failed test not deleted: %q", rootDir) return } t.Logf("Deleting temp directory %q", rootDir) assert.NoError(t, os.RemoveAll(rootDir)) }) return &integrationHarness{ ctx: context.Background(), logger: toolkit.NewTestLogger(t), rootDir: toolkit.Dir{Path: rootDir}, } } func (h *integrationHarness) mkDir(t *testing.T, name string) toolkit.Dir { fullName := fmt.Sprintf("%s-%d", name, atomic.AddUint64(&h.dirCounter, 1)) t.Logf("Creating directory %q", fullName) d, err := h.rootDir.MkChildDir(fullName, false) require.NoError(t, err) return d } type networkConfigOpts struct { hasPublicAddr bool numStorageAllocs int } func (o *networkConfigOpts) withDefaults() *networkConfigOpts { if o == nil { o = new(networkConfigOpts) } return o } func (h *integrationHarness) mkNetworkConfig( t *testing.T, opts *networkConfigOpts, ) daecommon.NetworkConfig { if opts == nil { opts = new(networkConfigOpts) } publicAddr := "" if opts.hasPublicAddr { publicAddr = newPublicAddr() } allocs := make([]map[string]any, opts.numStorageAllocs) for i := range allocs { allocs[i] = map[string]any{ "data_path": h.mkDir(t, "data").Path, "meta_path": h.mkDir(t, "meta").Path, "capacity": 1, } } allocsJSON, err := json.Marshal(allocs) require.NoError(t, err) return mustParseNetworkConfigf(` vpn: public_addr: %q tun: device: %q storage: allocations: %s `, publicAddr, newTunDevice(), allocsJSON, ) } type createNetworkOpts struct { creationParams bootstrap.CreationParams manualShutdown bool numStorageAllocs int } func (o *createNetworkOpts) withDefaults() *createNetworkOpts { if o == nil { o = new(createNetworkOpts) } if o.creationParams == (bootstrap.CreationParams{}) { o.creationParams = bootstrap.NewCreationParams("test", "test.localnet") } if o.numStorageAllocs == 0 { o.numStorageAllocs = 3 } return o } type integrationHarnessNetwork struct { Network ctx context.Context logger *mlog.Logger hostName nebula.HostName stateDir, runtimeDir toolkit.Dir opts *Opts } func (h *integrationHarness) createNetwork( t *testing.T, hostNameStr string, opts *createNetworkOpts, ) *integrationHarnessNetwork { t.Logf("Creating as %q", hostNameStr) opts = opts.withDefaults() var ( logger = h.logger.WithNamespace("network").WithNamespace(hostNameStr) networkConfig = h.mkNetworkConfig(t, &networkConfigOpts{ hasPublicAddr: true, numStorageAllocs: opts.numStorageAllocs, }) stateDir = h.mkDir(t, "state") runtimeDir = h.mkDir(t, "runtime") ipNet = newIPNet() hostName = nebula.HostName(hostNameStr) networkOpts = &Opts{ GarageAdminToken: "admin_token", Config: &networkConfig, } ) network, err := create( h.ctx, logger, getEnvBinDirPath(), stateDir, runtimeDir, opts.creationParams, ipNet, hostName, networkOpts, ) if err != nil { t.Fatalf("creating Network: %v", err) } nh := &integrationHarnessNetwork{ network, h.ctx, logger, hostName, stateDir, runtimeDir, networkOpts, } if !opts.manualShutdown { t.Cleanup(func() { t.Logf("Shutting down Network %q", hostNameStr) if err := nh.Shutdown(); err != nil { t.Logf("Shutting down Network %q failed: %v", hostNameStr, err) } }) } return nh } type joinNetworkOpts struct { *networkConfigOpts canCreateHosts bool manualShutdown bool } func (o *joinNetworkOpts) withDefaults() *joinNetworkOpts { if o == nil { o = new(joinNetworkOpts) } o.networkConfigOpts = o.networkConfigOpts.withDefaults() return o } func (h *integrationHarness) joinNetwork( t *testing.T, network *integrationHarnessNetwork, hostNameStr string, opts *joinNetworkOpts, ) *integrationHarnessNetwork { opts = opts.withDefaults() hostName := nebula.HostName(hostNameStr) t.Logf("Creating bootstrap for %q", hostNameStr) joiningBootstrap, err := network.CreateHost(h.ctx, hostName, CreateHostOpts{ CanCreateHosts: opts.canCreateHosts, }) if err != nil { t.Fatalf("creating host joining bootstrap: %v", err) } var ( logger = h.logger.WithNamespace("network").WithNamespace(hostNameStr) networkConfig = h.mkNetworkConfig(t, opts.networkConfigOpts) stateDir = h.mkDir(t, "state") runtimeDir = h.mkDir(t, "runtime") networkOpts = &Opts{ GarageAdminToken: "admin_token", Config: &networkConfig, } ) t.Logf("Joining as %q", hostNameStr) joinedNetwork, err := join( h.ctx, logger, getEnvBinDirPath(), joiningBootstrap, stateDir, runtimeDir, networkOpts, ) if err != nil { t.Fatalf("joining network: %v", err) } nh := &integrationHarnessNetwork{ joinedNetwork, h.ctx, logger, hostName, stateDir, runtimeDir, networkOpts, } if !opts.manualShutdown { t.Cleanup(func() { t.Logf("Shutting down Network %q", hostNameStr) if err := nh.Shutdown(); err != nil { t.Logf("Shutting down Network %q failed: %v", hostNameStr, err) } }) } return nh } func (nh *integrationHarnessNetwork) restart(t *testing.T) { t.Log("Shutting down network (restart)") require.NoError(t, nh.Network.Shutdown()) t.Log("Loading network (restart)") var err error nh.Network, err = load( nh.ctx, nh.logger, getEnvBinDirPath(), nh.stateDir, nh.runtimeDir, nh.opts, ) require.NoError(t, err) } func (nh *integrationHarnessNetwork) getConfig(t *testing.T) daecommon.NetworkConfig { networkConfig, err := nh.Network.GetConfig(nh.ctx) require.NoError(t, err) return networkConfig } func (nh *integrationHarnessNetwork) getBootstrap( t *testing.T, ) bootstrap.Bootstrap { currBootstrap, err := nh.Network.(*network).getBootstrap() require.NoError(t, err) return currBootstrap } func (nh *integrationHarnessNetwork) garageAdminClient( t *testing.T, ) *garage.AdminClient { c := newGarageAdminClient( nh.logger, nh.getConfig(t), nh.opts.GarageAdminToken, nh.getBootstrap(t).ThisHost(), ) t.Cleanup(func() { assert.NoError(t, c.Close()) }) return c } func (nh *integrationHarnessNetwork) getHostsByName( t *testing.T, ) map[nebula.HostName]bootstrap.Host { hosts, err := nh.Network.GetHosts(nh.ctx) require.NoError(t, err) hostsByName := map[nebula.HostName]bootstrap.Host{} for _, h := range hosts { hostsByName[h.Name] = h } return hostsByName }