package network import ( "fmt" "isle/bootstrap" "isle/daemon/daecommon" "isle/garage" "isle/garage/garagesrv" "isle/jsonutil" "isle/nebula" "isle/toolkit" "os" "path/filepath" "testing" "time" "github.com/stretchr/testify/assert" ) func TestCreate(t *testing.T) { var ( h = newIntegrationHarness(t) network = h.createNetwork(t, "primus", nil) ) gotCreationParams, err := loadCreationParams(network.stateDir) assert.NoError(t, err) assert.Equal( t, gotCreationParams, network.getBootstrap(t).NetworkCreationParams, ) } func TestLoad(t *testing.T) { t.Parallel() t.Run("given config", func(t *testing.T) { var ( h = newIntegrationHarness(t) network = h.createNetwork(t, "primus", nil) networkConfig = network.getConfig(t) ) network.opts.Config = &networkConfig network.restart(t) assert.Equal(t, networkConfig, network.getConfig(t)) }) t.Run("load previous config", func(t *testing.T) { var ( h = newIntegrationHarness(t) network = h.createNetwork(t, "primus", nil) networkConfig = network.getConfig(t) ) network.opts.Config = nil network.restart(t) assert.Equal(t, networkConfig, network.getConfig(t)) }) t.Run("garage lmdb db engine", func(t *testing.T) { var ( h = newIntegrationHarness(t) network = h.createNetwork(t, "primus", &createNetworkOpts{ garageDefaultDBEngine: garagesrv.DBEngineLMDB, }) metaPath = h.mkDir(t, "meta").Path ) h.logger.Info(h.ctx, "Checking that garage is using the expected db engine") garageConfig, err := os.ReadFile( filepath.Join(network.runtimeDir.Path, "garage-3900.toml"), ) assert.NoError(t, err) assert.Contains(t, string(garageConfig), `db_engine = "`+garagesrv.DBEngineLMDB+`"`, ) assert.NoFileExists(t, filepath.Join(metaPath, "db.sqlite")) network.opts.garageDefaultDBEngine = "" network.restart(t) h.logger.Info(h.ctx, "Checking that garage is still using the expected db engine") garageConfig, err = os.ReadFile( filepath.Join(network.runtimeDir.Path, "garage-3900.toml"), ) assert.NoError(t, err) assert.Contains(t, string(garageConfig), `db_engine = "`+garagesrv.DBEngineLMDB+`"`, ) assert.NoFileExists(t, filepath.Join(metaPath, "db.sqlite")) }) } func TestJoin(t *testing.T) { t.Parallel() t.Run("simple", func(t *testing.T) { var ( h = newIntegrationHarness(t) primus = h.createNetwork(t, "primus", nil) secondus = h.joinNetwork(t, primus, "secondus", nil) ) assert.Equal(t, primus.getHostsByName(t), secondus.getHostsByName(t)) }) t.Run("with alloc", func(t *testing.T) { var ( h = newIntegrationHarness(t) primus = h.createNetwork(t, "primus", nil) secondusBlocker = toolkit.NewTestBlocker(t) secondusRuntimeDir = h.mkDir(t, "runtime") ) secondusBlocker.Expect("Children.Reload.postReloadNebula").Then( t, h.ctx, func() { h.logger.Info(h.ctx, "Checking that firewall was updated with new alloc") assertFirewallInboundEquals( t, secondusRuntimeDir, []daecommon.ConfigFirewallRule{ {Port: "any", Proto: "icmp", Host: "any"}, {Port: "3900", Proto: "tcp", Host: "any"}, {Port: "3901", Proto: "tcp", Host: "any"}, }, ) }, ) secondus := h.joinNetwork(t, primus, "secondus", &joinNetworkOpts{ networkConfigOpts: &networkConfigOpts{ numStorageAllocs: 1, }, blocker: secondusBlocker, runtimeDir: secondusRuntimeDir, }) h.logger.Info(h.ctx, "reloading primus' hosts") assert.NoError(t, primus.Network.(*network).reloadHosts(h.ctx)) assert.Equal(t, primus.getHostsByName(t), secondus.getHostsByName(t)) assertGarageLayout(t, map[*integrationHarnessNetwork]int{ primus: 3, secondus: 1, }) }) } func TestNetwork_GetBootstrap(t *testing.T) { var ( h = newIntegrationHarness(t) network = h.createNetwork(t, "primus", nil) ) currBootstrap, err := network.GetBootstrap(h.ctx) assert.NoError(t, err) assert.Equal( t, nebula.HostPrivateCredentials{}, currBootstrap.PrivateCredentials, ) } func TestNetwork_CreateHost(t *testing.T) { t.Parallel() // Normal functionality of this method is tested as part of // `integrationHarness.joinNetwork`. This tests various extra behavior. t.Run("ErrIPInUse", func(t *testing.T) { var ( h = newIntegrationHarness(t) network = h.createNetwork(t, "primus", nil) hostName = nebula.HostName("secondus") ) _, err := network.CreateHost(h.ctx, hostName, CreateHostOpts{ IP: network.getBootstrap(t).ThisHost().IP(), }) assert.ErrorIs(t, err, ErrIPInUse) }) } func TestNetwork_SetConfig(t *testing.T) { t.Parallel() allocsToRoles := func( hostName nebula.HostName, allocs []bootstrap.GarageHostInstance, ) []garage.Role { roles := make([]garage.Role, len(allocs)) for i := range allocs { roles[i] = garage.Role{ ID: allocs[i].ID, Capacity: 1_000_000_000, Zone: string(hostName), Tags: []string{}, } } return roles } t.Run("add storage alloc/simple", func(t *testing.T) { var ( h = newIntegrationHarness(t) blocker = toolkit.NewTestBlocker(t) network = h.createNetwork(t, "primus", &createNetworkOpts{ blocker: blocker, }) networkConfig = network.getConfig(t) metaPath = h.mkDir(t, "meta").Path ) networkConfig.Storage.Allocations = append( networkConfig.Storage.Allocations, daecommon.ConfigStorageAllocation{ DataPath: h.mkDir(t, "data").Path, MetaPath: metaPath, Capacity: 1, S3APIPort: 4901, RPCPort: 4900, AdminPort: 4902, }, ) blocker.Expect("Children.Reload.postReloadNebula").Then( t, h.ctx, func() { h.logger.Info(h.ctx, "Checking that firewall was updated with new alloc") assertFirewallInboundEquals( t, network.runtimeDir, []daecommon.ConfigFirewallRule{ {Port: "any", Proto: "icmp", Host: "any"}, {Port: "3900", Proto: "tcp", Host: "any"}, {Port: "3901", Proto: "tcp", Host: "any"}, {Port: "3910", Proto: "tcp", Host: "any"}, {Port: "3911", Proto: "tcp", Host: "any"}, {Port: "3920", Proto: "tcp", Host: "any"}, {Port: "3921", Proto: "tcp", Host: "any"}, {Port: "4900", Proto: "tcp", Host: "any"}, {Port: "4901", Proto: "tcp", Host: "any"}, }, ) }, ) assert.NoError(t, network.SetConfig(h.ctx, networkConfig)) h.logger.Info(h.ctx, "Checking that the Host information was updated") newHostsByName := network.getHostsByName(t) newHost, ok := newHostsByName[network.hostName] assert.True(t, ok) allocs := newHost.HostConfigured.Garage.Instances assert.Len(t, allocs, 4) newAlloc := allocs[3] assert.NotEmpty(t, newAlloc.ID) newAlloc.ID = "" assert.Equal(t, bootstrap.GarageHostInstance{ S3APIPort: 4901, RPCPort: 4900, }, newAlloc) h.logger.Info(h.ctx, "Checking that the bootstrap file was written with the new host config") var storedBootstrap bootstrap.Bootstrap assert.NoError(t, jsonutil.LoadFile( &storedBootstrap, bootstrap.StateDirPath(network.stateDir.Path), )) assert.Equal(t, newHostsByName, storedBootstrap.Hosts) h.logger.Info(h.ctx, "Checking that garage layout contains the new allocation") expRoles := allocsToRoles(network.hostName, allocs) layout, err := network.garageAdminClient(t).GetLayout(h.ctx) assert.NoError(t, err) assert.ElementsMatch(t, expRoles, layout.Roles) h.logger.Info(h.ctx, "Checking that garage is using the expected db engine") garageConfig, err := os.ReadFile( filepath.Join(network.runtimeDir.Path, "garage-4900.toml"), ) assert.NoError(t, err) assert.Contains(t, string(garageConfig), `db_engine = "`+garagesrv.DBEngineSqlite+`"`, ) assert.FileExists(t, filepath.Join(metaPath, "db.sqlite")) }) t.Run("add storage alloc/on second host", func(t *testing.T) { var ( h = newIntegrationHarness(t) primus = h.createNetwork(t, "primus", nil) secondusBlocker = toolkit.NewTestBlocker(t) secondus = h.joinNetwork(t, primus, "secondus", &joinNetworkOpts{ blocker: secondusBlocker, }) secondusNetworkConfig = secondus.getConfig(t) ) secondusBlocker.Expect("Children.Reload.postReloadNebula").Then( t, h.ctx, func() { h.logger.Info(h.ctx, "Checking that firewall was updated with new alloc") assertFirewallInboundEquals( t, secondus.runtimeDir, []daecommon.ConfigFirewallRule{ {Port: "any", Proto: "icmp", Host: "any"}, {Port: "3900", Proto: "tcp", Host: "any"}, {Port: "3901", Proto: "tcp", Host: "any"}, }, ) }, ) secondusNetworkConfig.Storage.Allocations = append( secondusNetworkConfig.Storage.Allocations, daecommon.ConfigStorageAllocation{ DataPath: h.mkDir(t, "data").Path, MetaPath: h.mkDir(t, "meta").Path, Capacity: 1, S3APIPort: 3901, RPCPort: 3900, AdminPort: 3902, }, ) assert.NoError(t, secondus.SetConfig(h.ctx, secondusNetworkConfig)) assertGarageLayout(t, map[*integrationHarnessNetwork]int{ primus: 3, secondus: 1, }) }) t.Run("remove storage alloc", func(t *testing.T) { var ( h = newIntegrationHarness(t) network = h.createNetwork(t, "primus", &createNetworkOpts{ numStorageAllocs: 4, }) networkConfig = network.getConfig(t) prevHost = network.getHostsByName(t)[network.hostName] removedAlloc = networkConfig.Storage.Allocations[3] ) var removedGarageInst bootstrap.GarageHostInstance for _, removedGarageInst = range prevHost.Garage.Instances { if removedGarageInst.RPCPort == removedAlloc.RPCPort { break } } networkConfig.Storage.Allocations = networkConfig.Storage.Allocations[:3] assert.NoError(t, network.SetConfig(h.ctx, networkConfig)) h.logger.Info(h.ctx, "Checking that the Host information was updated") newHostsByName := network.getHostsByName(t) newHost, ok := newHostsByName[network.hostName] assert.True(t, ok) allocs := newHost.HostConfigured.Garage.Instances assert.Len(t, allocs, 3) assert.NotContains(t, allocs, removedGarageInst) h.logger.Info(h.ctx, "Checking that the bootstrap file was written with the new host config") var storedBootstrap bootstrap.Bootstrap assert.NoError(t, jsonutil.LoadFile( &storedBootstrap, bootstrap.StateDirPath(network.stateDir.Path), )) assert.Equal(t, newHostsByName, storedBootstrap.Hosts) h.logger.Info(h.ctx, "Checking that garage layout contains the new allocation") expRoles := allocsToRoles(network.hostName, allocs) layout, err := network.garageAdminClient(t).GetLayout(h.ctx) assert.NoError(t, err) assert.ElementsMatch(t, expRoles, layout.Roles) }) t.Run("changes reflected after restart", func(t *testing.T) { var ( h = newIntegrationHarness(t) network = h.createNetwork(t, "primus", &createNetworkOpts{ numStorageAllocs: 4, }) networkConfig = network.getConfig(t) ) networkConfig.Storage.Allocations = networkConfig.Storage.Allocations[:3] assert.NoError(t, network.SetConfig(h.ctx, networkConfig)) network.opts.Config = nil network.restart(t) assert.Equal(t, networkConfig, network.getConfig(t)) }) } func TestNetwork_glmStateTransition(t *testing.T) { var ( h = newIntegrationHarness(t) primus = h.createNetwork(t, "primus", nil) secondus = h.joinNetwork(t, primus, "secondus", &joinNetworkOpts{ networkConfigOpts: &networkConfigOpts{ numStorageAllocs: 1, }, }) secondusNetworkConfig = secondus.getConfig(t) secondusAdminClient = secondus.garageAdminClient(t) secondusNetworkDirect = secondus.Network.(*network) secondsBootstrapHost = primus.getHostsByName(t)["secondus"] ) assertGarageLayout(t, map[*integrationHarnessNetwork]int{ primus: 3, secondus: 1, }) secondusNetworkConfig.Storage.Allocations = nil assert.NoError(t, secondus.SetConfig(h.ctx, secondusNetworkConfig)) assert.Len(t, secondusNetworkDirect.children.ActiveStorageAllocations(), 1) h.logger.Info(h.ctx, "Waiting for secondus to finish draining") err := toolkit.UntilTrue( h.ctx, h.logger, 1*time.Second, func() (bool, error) { status, err := secondusAdminClient.Status(h.ctx) if err != nil { return false, fmt.Errorf("getting status: %w", err) } for _, node := range status.Nodes { if node.Addr.Addr() == secondsBootstrapHost.IP() { return !node.Draining, nil } } return false, fmt.Errorf("secondus not found in cluster status: %+v", status) }, ) assert.NoError(t, err) h.logger.Info(h.ctx, "Running GLM state transition") assert.NoError(t, secondusNetworkDirect.glmStateTransition(h.ctx)) assertGarageLayout(t, map[*integrationHarnessNetwork]int{ primus: 3, }) assert.Empty(t, secondusNetworkDirect.children.ActiveStorageAllocations()) }