isle/go/daemon/network/network_it_test.go

package network

import (
	"fmt"
	"isle/bootstrap"
	"isle/daemon/daecommon"
	"isle/garage"
	"isle/garage/garagesrv"
	"isle/jsonutil"
	"isle/nebula"
	"isle/toolkit"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestCreate(t *testing.T) {
	var (
		h       = newIntegrationHarness(t)
		network = h.createNetwork(t, "primus", nil)
	)

	gotCreationParams, err := loadCreationParams(network.stateDir)
	assert.NoError(t, err)
	assert.Equal(
		t, gotCreationParams, network.getBootstrap(t).NetworkCreationParams,
	)
}

func TestLoad(t *testing.T) {
	t.Parallel()

	t.Run("given config", func(t *testing.T) {
		var (
			h             = newIntegrationHarness(t)
			network       = h.createNetwork(t, "primus", nil)
			networkConfig = network.getConfig(t)
		)

		network.opts.Config = &networkConfig
		network.restart(t)

		assert.Equal(t, networkConfig, network.getConfig(t))
	})

	t.Run("load previous config", func(t *testing.T) {
		var (
			h             = newIntegrationHarness(t)
			network       = h.createNetwork(t, "primus", nil)
			networkConfig = network.getConfig(t)
		)

		network.opts.Config = nil
		network.restart(t)

		assert.Equal(t, networkConfig, network.getConfig(t))
	})

	t.Run("garage lmdb db engine", func(t *testing.T) {
		var (
			h       = newIntegrationHarness(t)
			network = h.createNetwork(t, "primus", &createNetworkOpts{
				garageDefaultDBEngine: garagesrv.DBEngineLMDB,
			})
			metaPath = h.mkDir(t, "meta").Path
		)

		h.logger.Info(h.ctx, "Checking that garage is using the expected db engine")
		garageConfig, err := os.ReadFile(
			filepath.Join(network.runtimeDir.Path, "garage-3900.toml"),
		)
		assert.NoError(t, err)
		assert.Contains(t,
			string(garageConfig),
			`db_engine = "`+garagesrv.DBEngineLMDB+`"`,
		)
		assert.NoFileExists(t, filepath.Join(metaPath, "db.sqlite"))

		network.opts.garageDefaultDBEngine = ""
		network.restart(t)

		h.logger.Info(h.ctx, "Checking that garage is still using the expected db engine")
		garageConfig, err = os.ReadFile(
			filepath.Join(network.runtimeDir.Path, "garage-3900.toml"),
		)
		assert.NoError(t, err)
		assert.Contains(t,
			string(garageConfig),
			`db_engine = "`+garagesrv.DBEngineLMDB+`"`,
		)
		assert.NoFileExists(t, filepath.Join(metaPath, "db.sqlite"))
	})

}

func TestJoin(t *testing.T) {
	t.Parallel()

	t.Run("simple", func(t *testing.T) {
		var (
			h        = newIntegrationHarness(t)
			primus   = h.createNetwork(t, "primus", nil)
			secondus = h.joinNetwork(t, primus, "secondus", nil)
		)

		assert.Equal(t, primus.getHostsByName(t), secondus.getHostsByName(t))
	})

	t.Run("with alloc/simple", func(t *testing.T) {
		var (
			h        = newIntegrationHarness(t)
			primus   = h.createNetwork(t, "primus", nil)
			secondus = h.joinNetwork(t, primus, "secondus", &joinNetworkOpts{
				networkConfigOpts: &networkConfigOpts{
					numStorageAllocs: 1,
				},
			})
		)

		h.logger.Info(h.ctx, "reloading primus' hosts")
		assert.NoError(t, primus.Network.(*network).reloadHosts(h.ctx))

		assert.Equal(t, primus.getHostsByName(t), secondus.getHostsByName(t))
		assertGarageLayout(t, map[*integrationHarnessNetwork]int{
			primus:   3,
			secondus: 1,
		})
	})
}

func TestNetwork_GetBootstrap(t *testing.T) {
	var (
		h       = newIntegrationHarness(t)
		network = h.createNetwork(t, "primus", nil)
	)

	currBootstrap, err := network.GetBootstrap(h.ctx)
	assert.NoError(t, err)
	assert.Equal(
		t, nebula.HostPrivateCredentials{}, currBootstrap.PrivateCredentials,
	)
}

func TestNetwork_CreateHost(t *testing.T) {
	t.Parallel()

	// Normal functionality of this method is tested as part of
	// `integrationHarness.joinNetwork`. This tests various extra behavior.

	t.Run("ErrIPInUse", func(t *testing.T) {
		var (
			h        = newIntegrationHarness(t)
			network  = h.createNetwork(t, "primus", nil)
			hostName = nebula.HostName("secondus")
		)

		_, err := network.CreateHost(h.ctx, hostName, CreateHostOpts{
			IP: network.getBootstrap(t).ThisHost().IP(),
		})
		assert.ErrorIs(t, err, ErrIPInUse)
	})
}

func TestNetwork_SetConfig(t *testing.T) {
	t.Parallel()

	allocsToRoles := func(
		hostName nebula.HostName, allocs []bootstrap.GarageHostInstance,
	) []garage.Role {
		roles := make([]garage.Role, len(allocs))
		for i := range allocs {
			roles[i] = garage.Role{
				ID:       allocs[i].ID,
				Capacity: 1_000_000_000,
				Zone:     string(hostName),
				Tags:     []string{},
			}
		}
		return roles
	}

	t.Run("add storage alloc/simple", func(t *testing.T) {
		var (
			h             = newIntegrationHarness(t)
			network       = h.createNetwork(t, "primus", nil)
			networkConfig = network.getConfig(t)
			metaPath      = h.mkDir(t, "meta").Path
		)

		networkConfig.Storage.Allocations = append(
			networkConfig.Storage.Allocations,
			daecommon.ConfigStorageAllocation{
				DataPath:  h.mkDir(t, "data").Path,
				MetaPath:  metaPath,
				Capacity:  1,
				S3APIPort: 4900,
				RPCPort:   4901,
				AdminPort: 4902,
			},
		)

		assert.NoError(t, network.SetConfig(h.ctx, networkConfig))

		h.logger.Info(h.ctx, "Checking that the Host information was updated")
		newHostsByName := network.getHostsByName(t)
		newHost, ok := newHostsByName[network.hostName]
		assert.True(t, ok)

		allocs := newHost.HostConfigured.Garage.Instances
		assert.Len(t, allocs, 4)

		newAlloc := allocs[3]
		assert.NotEmpty(t, newAlloc.ID)
		newAlloc.ID = ""
		assert.Equal(t, bootstrap.GarageHostInstance{
			S3APIPort: 4900,
			RPCPort:   4901,
		}, newAlloc)

		h.logger.Info(h.ctx, "Checking that the bootstrap file was written with the new host config")
		var storedBootstrap bootstrap.Bootstrap
		assert.NoError(t, jsonutil.LoadFile(
			&storedBootstrap, bootstrap.StateDirPath(network.stateDir.Path),
		))
		assert.Equal(t, newHostsByName, storedBootstrap.Hosts)

		h.logger.Info(h.ctx, "Checking that garage layout contains the new allocation")
		expRoles := allocsToRoles(network.hostName, allocs)
		layout, err := network.garageAdminClient(t).GetLayout(h.ctx)
		assert.NoError(t, err)
		assert.ElementsMatch(t, expRoles, layout.Roles)

		h.logger.Info(h.ctx, "Checking that garage is using the expected db engine")
		garageConfig, err := os.ReadFile(
			filepath.Join(network.runtimeDir.Path, "garage-4901.toml"),
		)
		assert.NoError(t, err)
		assert.Contains(t,
			string(garageConfig),
			`db_engine = "`+garagesrv.DBEngineSqlite+`"`,
		)
		assert.FileExists(t, filepath.Join(metaPath, "db.sqlite"))
	})

	t.Run("remove storage alloc", func(t *testing.T) {
		var (
			h       = newIntegrationHarness(t)
			network = h.createNetwork(t, "primus", &createNetworkOpts{
				numStorageAllocs: 4,
			})
			networkConfig = network.getConfig(t)

			prevHost     = network.getHostsByName(t)[network.hostName]
			removedAlloc = networkConfig.Storage.Allocations[3]
		)

		var removedGarageInst bootstrap.GarageHostInstance
		for _, removedGarageInst = range prevHost.Garage.Instances {
			if removedGarageInst.RPCPort == removedAlloc.RPCPort {
				break
			}
		}

		networkConfig.Storage.Allocations = networkConfig.Storage.Allocations[:3]
		assert.NoError(t, network.SetConfig(h.ctx, networkConfig))

		h.logger.Info(h.ctx, "Checking that the Host information was updated")
		newHostsByName := network.getHostsByName(t)
		newHost, ok := newHostsByName[network.hostName]
		assert.True(t, ok)

		allocs := newHost.HostConfigured.Garage.Instances
		assert.Len(t, allocs, 3)
		assert.NotContains(t, allocs, removedGarageInst)

		h.logger.Info(h.ctx, "Checking that the bootstrap file was written with the new host config")
		var storedBootstrap bootstrap.Bootstrap
		assert.NoError(t, jsonutil.LoadFile(
			&storedBootstrap, bootstrap.StateDirPath(network.stateDir.Path),
		))
		assert.Equal(t, newHostsByName, storedBootstrap.Hosts)

		h.logger.Info(h.ctx, "Checking that garage layout contains the new allocation")
		expRoles := allocsToRoles(network.hostName, allocs)
		layout, err := network.garageAdminClient(t).GetLayout(h.ctx)
		assert.NoError(t, err)
		assert.ElementsMatch(t, expRoles, layout.Roles)
	})

	t.Run("changes reflected after restart", func(t *testing.T) {
		var (
			h       = newIntegrationHarness(t)
			network = h.createNetwork(t, "primus", &createNetworkOpts{
				numStorageAllocs: 4,
			})
			networkConfig = network.getConfig(t)
		)

		networkConfig.Storage.Allocations = networkConfig.Storage.Allocations[:3]
		assert.NoError(t, network.SetConfig(h.ctx, networkConfig))

		network.opts.Config = nil
		network.restart(t)

		assert.Equal(t, networkConfig, network.getConfig(t))
	})
}

func TestNetwork_glmStateTransition(t *testing.T) {
	var (
		h        = newIntegrationHarness(t)
		primus   = h.createNetwork(t, "primus", nil)
		secondus = h.joinNetwork(t, primus, "secondus", &joinNetworkOpts{
			networkConfigOpts: &networkConfigOpts{
				numStorageAllocs: 1,
			},
		})
		secondusNetworkConfig = secondus.getConfig(t)
		secondusAdminClient   = secondus.garageAdminClient(t)
		secondusNetworkDirect = secondus.Network.(*network)
		secondsBootstrapHost  = primus.getHostsByName(t)["secondus"]
	)

	assertGarageLayout(t, map[*integrationHarnessNetwork]int{
		primus:   3,
		secondus: 1,
	})

	secondusNetworkConfig.Storage.Allocations = nil
	assert.NoError(t, secondus.SetConfig(h.ctx, secondusNetworkConfig))

	assert.Len(t, secondusNetworkDirect.children.ActiveStorageAllocations(), 1)

	h.logger.Info(h.ctx, "Waiting for secondus to finish draining")
	err := toolkit.UntilTrue(
		h.ctx, h.logger, 1*time.Second, func() (bool, error) {
			status, err := secondusAdminClient.Status(h.ctx)
			if err != nil {
				return false, fmt.Errorf("getting status: %w", err)
			}

			for _, node := range status.Nodes {
				if node.Addr.Addr() == secondsBootstrapHost.IP() {
					return !node.Draining, nil
				}
			}

			return false, fmt.Errorf("secondus not found in cluster status: %+v", status)
		},
	)
	assert.NoError(t, err)

	h.logger.Info(h.ctx, "Running GLM state transition")
	assert.NoError(t, secondusNetworkDirect.glmStateTransition(h.ctx))

	assertGarageLayout(t, map[*integrationHarnessNetwork]int{
		primus: 3,
	})
	assert.Empty(t, secondusNetworkDirect.children.ActiveStorageAllocations())
}