package network import ( "context" "encoding/json" "fmt" "io" "isle/bootstrap" "isle/daemon/children" "isle/daemon/daecommon" "isle/nebula" "isle/toolkit" "os" "path/filepath" "sync" "sync/atomic" "testing" "dev.mediocregopher.com/mediocre-go-lib.git/mlog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" ) // Utilities related to running network integration tests var ( getEnvBinDirPath = sync.OnceValue(func() string { appDirPath := os.Getenv("APPDIR") if appDirPath == "" { panic("APPDIR not set") } return filepath.Join(appDirPath, "bin") }) ipNetCounter uint64 = 0 publicAddrPortCounter uint64 = 1024 tunDeviceCounter uint64 = 0 ) func newIPNet() nebula.IPNet { var ( ipNet nebula.IPNet ipNetStr = fmt.Sprintf( "172.16.%d.0/24", atomic.AddUint64(&ipNetCounter, 1), ) ) if err := ipNet.UnmarshalText([]byte(ipNetStr)); err != nil { panic(fmt.Sprintf("parsing IPNet from %q: %v", ipNetStr, err)) } return ipNet } func newPublicAddr() string { return fmt.Sprintf( "127.0.0.200:%d", atomic.AddUint64(&publicAddrPortCounter, 1), ) } func newTunDevice() string { return fmt.Sprintf("isle-test-%d", atomic.AddUint64(&tunDeviceCounter, 1)) } func mustParseNetworkConfigf(str string, args ...any) daecommon.NetworkConfig { str = fmt.Sprintf(str, args...) var networkConfig daecommon.NetworkConfig if err := yaml.Unmarshal([]byte(str), &networkConfig); err != nil { panic(fmt.Sprintf("parsing network config: %v", err)) } return networkConfig } type integrationHarness struct { ctx context.Context logger *mlog.Logger rootDir toolkit.Dir dirCounter uint64 } func newIntegrationHarness(t *testing.T) *integrationHarness { t.Parallel() toolkit.MarkIntegrationTest(t) rootDir, err := os.MkdirTemp("", "isle-network-it-test.*") require.NoError(t, err) t.Logf("Temporary test directory: %q", rootDir) t.Cleanup(func() { if t.Failed() { t.Logf("Temp directory for failed test not deleted: %q", rootDir) return } t.Logf("Deleting temp directory %q", rootDir) assert.NoError(t, os.RemoveAll(rootDir)) }) return &integrationHarness{ ctx: context.Background(), logger: mlog.NewLogger(&mlog.LoggerOpts{ MessageHandler: mlog.NewTestMessageHandler(t), }), rootDir: toolkit.Dir{Path: rootDir}, } } func (h *integrationHarness) mkDir(t *testing.T, name string) toolkit.Dir { fullName := fmt.Sprintf("%s-%d", name, atomic.AddUint64(&h.dirCounter, 1)) t.Logf("Creating directory %q", fullName) d, err := h.rootDir.MkChildDir(fullName, false) require.NoError(t, err) return d } type networkConfigOpts struct { hasPublicAddr bool numStorageAllocs int } func (h *integrationHarness) mkNetworkConfig( t *testing.T, opts *networkConfigOpts, ) daecommon.NetworkConfig { if opts == nil { opts = new(networkConfigOpts) } publicAddr := "" if opts.hasPublicAddr { publicAddr = newPublicAddr() } allocs := make([]map[string]any, opts.numStorageAllocs) for i := range allocs { allocs[i] = map[string]any{ "data_path": h.mkDir(t, "data").Path, "meta_path": h.mkDir(t, "meta").Path, "capacity": 1, } } allocsJSON, err := json.Marshal(allocs) require.NoError(t, err) return mustParseNetworkConfigf(` vpn: public_addr: %q tun: device: %q storage: allocations: %s `, publicAddr, newTunDevice(), allocsJSON, ) } func (h *integrationHarness) mkChildrenOpts( t *testing.T, runtimeDir toolkit.Dir, ) *children.Opts { var ( childrenLogFilePath = filepath.Join(runtimeDir.Path, "children.log") childrenOpts children.Opts ) childrenLogFile, err := os.Create(childrenLogFilePath) require.NoError(t, err) t.Cleanup(func() { assert.NoError(t, err) }) if os.Getenv("ISLE_INTEGRATION_TEST_CHILDREN_LOG_STDOUT") == "" { childrenOpts = children.Opts{ Stdout: childrenLogFile, Stderr: childrenLogFile, } } else { childrenOpts = children.Opts{ Stdout: io.MultiWriter(os.Stdout, childrenLogFile), Stderr: io.MultiWriter(os.Stdout, childrenLogFile), } } return &childrenOpts } type createNetworkOpts struct { creationParams bootstrap.CreationParams manualShutdown bool } func (o *createNetworkOpts) withDefaults() *createNetworkOpts { if o == nil { o = new(createNetworkOpts) } if o.creationParams == (bootstrap.CreationParams{}) { o.creationParams = bootstrap.NewCreationParams("test", "test.localnet") } return o } type integrationHarnessNetwork struct { Network hostName nebula.HostName creationParams bootstrap.CreationParams networkConfig daecommon.NetworkConfig stateDir, runtimeDir toolkit.Dir opts *Opts } func (h *integrationHarness) createNetwork( t *testing.T, hostNameStr string, opts *createNetworkOpts, ) integrationHarnessNetwork { t.Logf("Creating as %q", hostNameStr) opts = opts.withDefaults() var ( networkConfig = h.mkNetworkConfig(t, &networkConfigOpts{ hasPublicAddr: true, numStorageAllocs: 3, }) stateDir = h.mkDir(t, "state") runtimeDir = h.mkDir(t, "runtime") ipNet = newIPNet() hostName = nebula.HostName(hostNameStr) networkOpts = &Opts{ ChildrenOpts: h.mkChildrenOpts(t, runtimeDir), GarageAdminToken: "admin_token", } ) network, err := Create( h.ctx, h.logger.WithNamespace("network").WithNamespace(hostNameStr), networkConfig, getEnvBinDirPath(), stateDir, runtimeDir, opts.creationParams, ipNet, hostName, networkOpts, ) if err != nil { t.Fatalf("creating Network: %v", err) } if !opts.manualShutdown { t.Cleanup(func() { t.Logf("Shutting down Network %q", hostNameStr) if err := network.Shutdown(); err != nil { t.Logf("Shutting down Network %q failed: %v", hostNameStr, err) } }) } return integrationHarnessNetwork{ network, hostName, opts.creationParams, networkConfig, stateDir, runtimeDir, networkOpts, } } type joinNetworkOpts struct { networkConfigOpts canCreateHosts bool manualShutdown bool } func (h *integrationHarness) joinNetwork( t *testing.T, network integrationHarnessNetwork, hostNameStr string, opts *joinNetworkOpts, ) integrationHarnessNetwork { t.Logf("Joining as %q", hostNameStr) opts = new(joinNetworkOpts) var ( networkConfig = h.mkNetworkConfig(t, &opts.networkConfigOpts) stateDir = h.mkDir(t, "state") runtimeDir = h.mkDir(t, "runtime") hostName = nebula.HostName(hostNameStr) networkOpts = &Opts{ ChildrenOpts: h.mkChildrenOpts(t, runtimeDir), GarageAdminToken: "admin_token", } ) joiningBootstrap, err := network.CreateHost(h.ctx, hostName, CreateHostOpts{ CanCreateHosts: opts.canCreateHosts, }) if err != nil { t.Fatalf("creating host joining bootstrap: %v", err) } joinedNetwork, err := Join( h.ctx, h.logger.WithNamespace("network").WithNamespace(hostNameStr), networkConfig, joiningBootstrap, getEnvBinDirPath(), stateDir, runtimeDir, networkOpts, ) if err != nil { t.Fatalf("joining network: %v", err) } if !opts.manualShutdown { t.Cleanup(func() { t.Logf("Shutting down Network %q", hostNameStr) if err := joinedNetwork.Shutdown(); err != nil { t.Logf("Shutting down Network %q failed: %v", hostNameStr, err) } }) } return integrationHarnessNetwork{ joinedNetwork, hostName, network.creationParams, networkConfig, stateDir, runtimeDir, networkOpts, } }