isle/go/daemon/network/network_it_test.go

448 lines
13 KiB
Go
Raw Permalink Normal View History

package network
import (
"fmt"
"isle/bootstrap"
"isle/daemon/daecommon"
"isle/garage"
"isle/garage/garagesrv"
"isle/jsonutil"
"isle/nebula"
"isle/toolkit"
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestCreate(t *testing.T) {
var (
h = newIntegrationHarness(t)
network = h.createNetwork(t, "primus", nil)
)
gotCreationParams, err := loadCreationParams(network.stateDir)
assert.NoError(t, err)
assert.Equal(
t, gotCreationParams, network.getBootstrap(t).NetworkCreationParams,
)
}
2024-10-07 20:41:46 +00:00
func TestLoad(t *testing.T) {
2024-11-12 12:42:35 +00:00
t.Parallel()
t.Run("given config", func(t *testing.T) {
var (
h = newIntegrationHarness(t)
network = h.createNetwork(t, "primus", nil)
networkConfig = network.getConfig(t)
)
2024-10-07 20:41:46 +00:00
network.opts.Config = &networkConfig
network.restart(t)
assert.Equal(t, networkConfig, network.getConfig(t))
})
t.Run("load previous config", func(t *testing.T) {
var (
h = newIntegrationHarness(t)
network = h.createNetwork(t, "primus", nil)
networkConfig = network.getConfig(t)
)
network.opts.Config = nil
network.restart(t)
assert.Equal(t, networkConfig, network.getConfig(t))
2024-10-07 20:41:46 +00:00
})
2025-01-07 14:40:50 +00:00
t.Run("garage lmdb db engine", func(t *testing.T) {
var (
h = newIntegrationHarness(t)
network = h.createNetwork(t, "primus", &createNetworkOpts{
garageDefaultDBEngine: garagesrv.DBEngineLMDB,
})
metaPath = h.mkDir(t, "meta").Path
)
h.logger.Info(h.ctx, "Checking that garage is using the expected db engine")
garageConfig, err := os.ReadFile(
filepath.Join(network.runtimeDir.Path, "garage-3900.toml"),
)
assert.NoError(t, err)
assert.Contains(t,
string(garageConfig),
`db_engine = "`+garagesrv.DBEngineLMDB+`"`,
)
assert.NoFileExists(t, filepath.Join(metaPath, "db.sqlite"))
network.opts.garageDefaultDBEngine = ""
network.restart(t)
h.logger.Info(h.ctx, "Checking that garage is still using the expected db engine")
garageConfig, err = os.ReadFile(
filepath.Join(network.runtimeDir.Path, "garage-3900.toml"),
)
assert.NoError(t, err)
assert.Contains(t,
string(garageConfig),
`db_engine = "`+garagesrv.DBEngineLMDB+`"`,
)
assert.NoFileExists(t, filepath.Join(metaPath, "db.sqlite"))
})
2024-10-07 20:41:46 +00:00
}
2024-10-14 10:12:43 +00:00
func TestJoin(t *testing.T) {
2024-11-12 12:42:35 +00:00
t.Parallel()
t.Run("simple", func(t *testing.T) {
var (
h = newIntegrationHarness(t)
primus = h.createNetwork(t, "primus", nil)
secondus = h.joinNetwork(t, primus, "secondus", nil)
)
2024-10-14 10:12:43 +00:00
assert.Equal(t, primus.getHostsByName(t), secondus.getHostsByName(t))
})
t.Run("with alloc", func(t *testing.T) {
var (
h = newIntegrationHarness(t)
primus = h.createNetwork(t, "primus", nil)
secondusBlocker = toolkit.NewTestBlocker(t)
secondusRuntimeDir = h.mkDir(t, "runtime")
)
secondusBlocker.Expect("Children.Reload.postReloadNebula").Then(
t, h.ctx, func() {
h.logger.Info(h.ctx, "Checking that firewall was updated with new alloc")
assertFirewallInboundEquals(
t, secondusRuntimeDir, []daecommon.ConfigFirewallRule{
{Port: "any", Proto: "icmp", Host: "any"},
{Port: "3900", Proto: "tcp", Host: "any"},
{Port: "3901", Proto: "tcp", Host: "any"},
},
)
},
)
secondus := h.joinNetwork(t, primus, "secondus", &joinNetworkOpts{
networkConfigOpts: &networkConfigOpts{
numStorageAllocs: 1,
},
blocker: secondusBlocker,
runtimeDir: secondusRuntimeDir,
})
2025-01-07 14:40:50 +00:00
h.logger.Info(h.ctx, "reloading primus' hosts")
assert.NoError(t, primus.Network.(*network).reloadHosts(h.ctx))
assert.Equal(t, primus.getHostsByName(t), secondus.getHostsByName(t))
assertGarageLayout(t, map[*integrationHarnessNetwork]int{
primus: 3,
secondus: 1,
})
})
2024-10-14 10:12:43 +00:00
}
2024-10-23 18:18:11 +00:00
func TestNetwork_GetBootstrap(t *testing.T) {
var (
h = newIntegrationHarness(t)
network = h.createNetwork(t, "primus", nil)
)
currBootstrap, err := network.GetBootstrap(h.ctx)
assert.NoError(t, err)
assert.Equal(
t, nebula.HostPrivateCredentials{}, currBootstrap.PrivateCredentials,
)
}
2024-12-12 19:20:27 +00:00
func TestNetwork_CreateHost(t *testing.T) {
t.Parallel()
// Normal functionality of this method is tested as part of
// `integrationHarness.joinNetwork`. This tests various extra behavior.
t.Run("ErrIPInUse", func(t *testing.T) {
var (
h = newIntegrationHarness(t)
network = h.createNetwork(t, "primus", nil)
hostName = nebula.HostName("secondus")
)
_, err := network.CreateHost(h.ctx, hostName, CreateHostOpts{
IP: network.getBootstrap(t).ThisHost().IP(),
})
assert.ErrorIs(t, err, ErrIPInUse)
})
}
func TestNetwork_SetConfig(t *testing.T) {
2024-11-12 12:42:35 +00:00
t.Parallel()
allocsToRoles := func(
hostName nebula.HostName, allocs []bootstrap.GarageHostInstance,
) []garage.Role {
roles := make([]garage.Role, len(allocs))
for i := range allocs {
roles[i] = garage.Role{
ID: allocs[i].ID,
Capacity: 1_000_000_000,
Zone: string(hostName),
Tags: []string{},
}
}
return roles
}
t.Run("add storage alloc/simple", func(t *testing.T) {
var (
h = newIntegrationHarness(t)
blocker = toolkit.NewTestBlocker(t)
network = h.createNetwork(t, "primus", &createNetworkOpts{
blocker: blocker,
})
networkConfig = network.getConfig(t)
metaPath = h.mkDir(t, "meta").Path
)
networkConfig.Storage.Allocations = append(
networkConfig.Storage.Allocations,
daecommon.ConfigStorageAllocation{
DataPath: h.mkDir(t, "data").Path,
MetaPath: metaPath,
Capacity: 1,
S3APIPort: 4901,
RPCPort: 4900,
AdminPort: 4902,
},
)
blocker.Expect("Children.Reload.postReloadNebula").Then(
t, h.ctx, func() {
h.logger.Info(h.ctx, "Checking that firewall was updated with new alloc")
assertFirewallInboundEquals(
t, network.runtimeDir, []daecommon.ConfigFirewallRule{
{Port: "any", Proto: "icmp", Host: "any"},
{Port: "3900", Proto: "tcp", Host: "any"},
{Port: "3901", Proto: "tcp", Host: "any"},
{Port: "3910", Proto: "tcp", Host: "any"},
{Port: "3911", Proto: "tcp", Host: "any"},
{Port: "3920", Proto: "tcp", Host: "any"},
{Port: "3921", Proto: "tcp", Host: "any"},
{Port: "4900", Proto: "tcp", Host: "any"},
{Port: "4901", Proto: "tcp", Host: "any"},
},
)
},
)
assert.NoError(t, network.SetConfig(h.ctx, networkConfig))
2025-01-07 14:40:50 +00:00
h.logger.Info(h.ctx, "Checking that the Host information was updated")
newHostsByName := network.getHostsByName(t)
newHost, ok := newHostsByName[network.hostName]
assert.True(t, ok)
allocs := newHost.HostConfigured.Garage.Instances
assert.Len(t, allocs, 4)
newAlloc := allocs[3]
assert.NotEmpty(t, newAlloc.ID)
newAlloc.ID = ""
assert.Equal(t, bootstrap.GarageHostInstance{
S3APIPort: 4901,
RPCPort: 4900,
}, newAlloc)
2025-01-07 14:40:50 +00:00
h.logger.Info(h.ctx, "Checking that the bootstrap file was written with the new host config")
var storedBootstrap bootstrap.Bootstrap
assert.NoError(t, jsonutil.LoadFile(
&storedBootstrap, bootstrap.StateDirPath(network.stateDir.Path),
))
assert.Equal(t, newHostsByName, storedBootstrap.Hosts)
2025-01-07 14:40:50 +00:00
h.logger.Info(h.ctx, "Checking that garage layout contains the new allocation")
expRoles := allocsToRoles(network.hostName, allocs)
layout, err := network.garageAdminClient(t).GetLayout(h.ctx)
assert.NoError(t, err)
assert.ElementsMatch(t, expRoles, layout.Roles)
2025-01-07 14:40:50 +00:00
h.logger.Info(h.ctx, "Checking that garage is using the expected db engine")
garageConfig, err := os.ReadFile(
filepath.Join(network.runtimeDir.Path, "garage-4900.toml"),
)
assert.NoError(t, err)
assert.Contains(t,
string(garageConfig),
`db_engine = "`+garagesrv.DBEngineSqlite+`"`,
)
assert.FileExists(t, filepath.Join(metaPath, "db.sqlite"))
})
t.Run("add storage alloc/on second host", func(t *testing.T) {
var (
h = newIntegrationHarness(t)
primus = h.createNetwork(t, "primus", nil)
secondusBlocker = toolkit.NewTestBlocker(t)
secondus = h.joinNetwork(t, primus, "secondus", &joinNetworkOpts{
blocker: secondusBlocker,
})
secondusNetworkConfig = secondus.getConfig(t)
)
secondusBlocker.Expect("Children.Reload.postReloadNebula").Then(
t, h.ctx, func() {
h.logger.Info(h.ctx, "Checking that firewall was updated with new alloc")
assertFirewallInboundEquals(
t, secondus.runtimeDir, []daecommon.ConfigFirewallRule{
{Port: "any", Proto: "icmp", Host: "any"},
{Port: "3900", Proto: "tcp", Host: "any"},
{Port: "3901", Proto: "tcp", Host: "any"},
},
)
},
)
secondusNetworkConfig.Storage.Allocations = append(
secondusNetworkConfig.Storage.Allocations,
daecommon.ConfigStorageAllocation{
DataPath: h.mkDir(t, "data").Path,
MetaPath: h.mkDir(t, "meta").Path,
Capacity: 1,
S3APIPort: 3901,
RPCPort: 3900,
AdminPort: 3902,
},
)
assert.NoError(t, secondus.SetConfig(h.ctx, secondusNetworkConfig))
assertGarageLayout(t, map[*integrationHarnessNetwork]int{
primus: 3,
secondus: 1,
})
})
t.Run("remove storage alloc", func(t *testing.T) {
var (
h = newIntegrationHarness(t)
network = h.createNetwork(t, "primus", &createNetworkOpts{
numStorageAllocs: 4,
})
networkConfig = network.getConfig(t)
2025-01-07 14:40:50 +00:00
prevHost = network.getHostsByName(t)[network.hostName]
removedAlloc = networkConfig.Storage.Allocations[3]
2024-10-23 18:18:11 +00:00
)
2025-01-07 14:40:50 +00:00
var removedGarageInst bootstrap.GarageHostInstance
for _, removedGarageInst = range prevHost.Garage.Instances {
if removedGarageInst.RPCPort == removedAlloc.RPCPort {
break
}
}
networkConfig.Storage.Allocations = networkConfig.Storage.Allocations[:3]
assert.NoError(t, network.SetConfig(h.ctx, networkConfig))
2025-01-07 14:40:50 +00:00
h.logger.Info(h.ctx, "Checking that the Host information was updated")
newHostsByName := network.getHostsByName(t)
newHost, ok := newHostsByName[network.hostName]
assert.True(t, ok)
allocs := newHost.HostConfigured.Garage.Instances
assert.Len(t, allocs, 3)
assert.NotContains(t, allocs, removedGarageInst)
2025-01-07 14:40:50 +00:00
h.logger.Info(h.ctx, "Checking that the bootstrap file was written with the new host config")
var storedBootstrap bootstrap.Bootstrap
assert.NoError(t, jsonutil.LoadFile(
&storedBootstrap, bootstrap.StateDirPath(network.stateDir.Path),
))
assert.Equal(t, newHostsByName, storedBootstrap.Hosts)
2025-01-07 14:40:50 +00:00
h.logger.Info(h.ctx, "Checking that garage layout contains the new allocation")
expRoles := allocsToRoles(network.hostName, allocs)
layout, err := network.garageAdminClient(t).GetLayout(h.ctx)
assert.NoError(t, err)
assert.ElementsMatch(t, expRoles, layout.Roles)
})
t.Run("changes reflected after restart", func(t *testing.T) {
var (
h = newIntegrationHarness(t)
network = h.createNetwork(t, "primus", &createNetworkOpts{
numStorageAllocs: 4,
})
networkConfig = network.getConfig(t)
)
networkConfig.Storage.Allocations = networkConfig.Storage.Allocations[:3]
assert.NoError(t, network.SetConfig(h.ctx, networkConfig))
network.opts.Config = nil
network.restart(t)
assert.Equal(t, networkConfig, network.getConfig(t))
})
2024-10-23 18:18:11 +00:00
}
2025-01-07 14:40:50 +00:00
func TestNetwork_glmStateTransition(t *testing.T) {
var (
h = newIntegrationHarness(t)
primus = h.createNetwork(t, "primus", nil)
secondus = h.joinNetwork(t, primus, "secondus", &joinNetworkOpts{
networkConfigOpts: &networkConfigOpts{
numStorageAllocs: 1,
},
})
secondusNetworkConfig = secondus.getConfig(t)
secondusAdminClient = secondus.garageAdminClient(t)
secondusNetworkDirect = secondus.Network.(*network)
secondsBootstrapHost = primus.getHostsByName(t)["secondus"]
)
assertGarageLayout(t, map[*integrationHarnessNetwork]int{
primus: 3,
secondus: 1,
})
secondusNetworkConfig.Storage.Allocations = nil
assert.NoError(t, secondus.SetConfig(h.ctx, secondusNetworkConfig))
assert.Len(t, secondusNetworkDirect.children.ActiveStorageAllocations(), 1)
h.logger.Info(h.ctx, "Waiting for secondus to finish draining")
err := toolkit.UntilTrue(
h.ctx, h.logger, 1*time.Second, func() (bool, error) {
status, err := secondusAdminClient.Status(h.ctx)
if err != nil {
return false, fmt.Errorf("getting status: %w", err)
}
for _, node := range status.Nodes {
if node.Addr.Addr() == secondsBootstrapHost.IP() {
return !node.Draining, nil
}
}
return false, fmt.Errorf("secondus not found in cluster status: %+v", status)
},
)
assert.NoError(t, err)
h.logger.Info(h.ctx, "Running GLM state transition")
assert.NoError(t, secondusNetworkDirect.glmStateTransition(h.ctx))
assertGarageLayout(t, map[*integrationHarnessNetwork]int{
primus: 3,
})
assert.Empty(t, secondusNetworkDirect.children.ActiveStorageAllocations())
}