isle/go/daemon/network/network_it_test.go

package network

import (
	"fmt"
	"isle/bootstrap"
	"isle/daemon/daecommon"
	"isle/garage"
	"isle/garage/garagesrv"
	"isle/jsonutil"
	"isle/nebula"
	"isle/toolkit"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestCreate(t *testing.T) {
	var (
		h       = newIntegrationHarness(t)
		network = h.createNetwork(t, "primus", nil)
	)

	gotCreationParams, err := loadCreationParams(network.stateDir)
	assert.NoError(t, err)
	assert.Equal(
		t, gotCreationParams, network.getBootstrap(t).NetworkCreationParams,
	)
}

func TestLoad(t *testing.T) {
	t.Parallel()

	t.Run("given config", func(t *testing.T) {
		var (
			h             = newIntegrationHarness(t)
			network       = h.createNetwork(t, "primus", nil)
			networkConfig = network.getConfig(t)
		)

		network.opts.Config = &networkConfig
		network.restart(t)

		assert.Equal(t, networkConfig, network.getConfig(t))
	})

	t.Run("load previous config", func(t *testing.T) {
		var (
			h             = newIntegrationHarness(t)
			network       = h.createNetwork(t, "primus", nil)
			networkConfig = network.getConfig(t)
		)

		network.opts.Config = nil
		network.restart(t)

		assert.Equal(t, networkConfig, network.getConfig(t))
	})
}

func TestJoin(t *testing.T) {
	t.Parallel()

	t.Run("simple", func(t *testing.T) {
		var (
			h        = newIntegrationHarness(t)
			primus   = h.createNetwork(t, "primus", nil)
			secondus = h.joinNetwork(t, primus, "secondus", nil)
		)

		assert.Equal(t, primus.getHostsByName(t), secondus.getHostsByName(t))
	})

	t.Run("with alloc/simple", func(t *testing.T) {
		var (
			h        = newIntegrationHarness(t)
			primus   = h.createNetwork(t, "primus", nil)
			secondus = h.joinNetwork(t, primus, "secondus", &joinNetworkOpts{
				networkConfigOpts: &networkConfigOpts{
					numStorageAllocs: 1,
				},
			})
		)

		t.Log("reloading primus' hosts")
		assert.NoError(t, primus.Network.(*network).reloadHosts(h.ctx))

		assert.Equal(t, primus.getHostsByName(t), secondus.getHostsByName(t))
		assertGarageLayout(t, map[*integrationHarnessNetwork]int{
			primus:   3,
			secondus: 1,
		})
	})

	// Assert that if primus runs the orphan remover at the same moment that
	// secondus is joining that the layout applied by secondus doesn't get
	// overwritten.
	t.Run("with alloc/remove orphans after garage layout applied", func(t *testing.T) {
		t.Skip("This is currently expected to fail. Orphan removal is going to be reworked accordingly")

		var (
			h                 = newIntegrationHarness(t)
			primus            = h.createNetwork(t, "primus", nil)
			primusAdminClient = primus.garageAdminClient(t)
			secondusBlocker   = toolkit.NewTestBlocker(t)
		)

		secondusBlocker.ExpectBlockpoint("garageLayoutApplied").On(
			t, h.ctx, func() {
				h.logger.Info(h.ctx, "Waiting for new layout to propagate to primus")
				err := toolkit.UntilTrue(
					h.ctx, h.logger, 1*time.Second, func() (bool, error) {
						layout, err := primusAdminClient.GetLayout(h.ctx)
						if err != nil {
							return false, fmt.Errorf("getting layout: %w", err)
						}

						return len(layout.Roles) == 4, nil
					},
				)

				if !assert.NoError(t, err) {
					return
				}

				//h.logger.Info(h.ctx, "Calling removeOrphanGarageNodes")
				//assert.NoError(
				//	t, primus.Network.(*network).removeOrphanGarageNodes(h.ctx),
				//)
			},
		)

		secondus := h.joinNetwork(t, primus, "secondus", &joinNetworkOpts{
			networkConfigOpts: &networkConfigOpts{
				numStorageAllocs: 1,
			},
			blocker: secondusBlocker,
		})

		assertGarageLayout(t, map[*integrationHarnessNetwork]int{
			primus:   3,
			secondus: 1,
		})
	})
}

func TestNetwork_GetBootstrap(t *testing.T) {
	var (
		h       = newIntegrationHarness(t)
		network = h.createNetwork(t, "primus", nil)
	)

	currBootstrap, err := network.GetBootstrap(h.ctx)
	assert.NoError(t, err)
	assert.Equal(
		t, nebula.HostPrivateCredentials{}, currBootstrap.PrivateCredentials,
	)
}

func TestNetwork_CreateHost(t *testing.T) {
	t.Parallel()

	// Normal functionality of this method is tested as part of
	// `integrationHarness.joinNetwork`. This tests various extra behavior.

	t.Run("ErrIPInUse", func(t *testing.T) {
		var (
			h        = newIntegrationHarness(t)
			network  = h.createNetwork(t, "primus", nil)
			hostName = nebula.HostName("secondus")
		)

		_, err := network.CreateHost(h.ctx, hostName, CreateHostOpts{
			IP: network.getBootstrap(t).ThisHost().IP(),
		})
		assert.ErrorIs(t, err, ErrIPInUse)
	})
}

func TestNetwork_SetConfig(t *testing.T) {
	t.Parallel()

	allocsToRoles := func(
		hostName nebula.HostName, allocs []bootstrap.GarageHostInstance,
	) []garage.Role {
		roles := make([]garage.Role, len(allocs))
		for i := range allocs {
			roles[i] = garage.Role{
				ID:       allocs[i].ID,
				Capacity: 1_000_000_000,
				Zone:     string(hostName),
				Tags:     []string{},
			}
		}
		return roles
	}

	t.Run("add storage alloc/simple", func(t *testing.T) {
		var (
			h             = newIntegrationHarness(t)
			network       = h.createNetwork(t, "primus", nil)
			networkConfig = network.getConfig(t)
			metaPath      = h.mkDir(t, "meta").Path
		)

		networkConfig.Storage.Allocations = append(
			networkConfig.Storage.Allocations,
			daecommon.ConfigStorageAllocation{
				DataPath:  h.mkDir(t, "data").Path,
				MetaPath:  metaPath,
				Capacity:  1,
				S3APIPort: 4900,
				RPCPort:   4901,
				AdminPort: 4902,
			},
		)

		assert.NoError(t, network.SetConfig(h.ctx, networkConfig))

		t.Log("Checking that the Host information was updated")
		newHostsByName := network.getHostsByName(t)
		newHost, ok := newHostsByName[network.hostName]
		assert.True(t, ok)

		allocs := newHost.HostConfigured.Garage.Instances
		assert.Len(t, allocs, 4)

		newAlloc := allocs[3]
		assert.NotEmpty(t, newAlloc.ID)
		newAlloc.ID = ""
		assert.Equal(t, bootstrap.GarageHostInstance{
			S3APIPort: 4900,
			RPCPort:   4901,
		}, newAlloc)

		t.Log("Checking that the bootstrap file was written with the new host config")
		var storedBootstrap bootstrap.Bootstrap
		assert.NoError(t, jsonutil.LoadFile(
			&storedBootstrap, bootstrap.StateDirPath(network.stateDir.Path),
		))
		assert.Equal(t, newHostsByName, storedBootstrap.Hosts)

		t.Log("Checking that garage layout contains the new allocation")
		expRoles := allocsToRoles(network.hostName, allocs)
		layout, err := network.garageAdminClient(t).GetLayout(h.ctx)
		assert.NoError(t, err)
		assert.ElementsMatch(t, expRoles, layout.Roles)

		t.Log("Checking that garage is using the expected db engine")
		garageConfig, err := os.ReadFile(
			filepath.Join(network.runtimeDir.Path, "garage-4901.toml"),
		)
		assert.NoError(t, err)
		assert.Contains(t,
			string(garageConfig),
			`db_engine = "`+garagesrv.DBEngineSqlite+`"`,
		)
		assert.FileExists(t, filepath.Join(metaPath, "db.sqlite"))
	})

	t.Run("add storage alloc/lmdb", func(t *testing.T) {
		var (
			h             = newIntegrationHarness(t)
			network       = h.createNetwork(t, "primus", nil)
			networkConfig = network.getConfig(t)
			dataPath      = h.mkDir(t, "data").Path
			metaPath      = h.mkDir(t, "meta").Path
		)

		networkConfig.Storage.Allocations = append(
			networkConfig.Storage.Allocations,
			daecommon.ConfigStorageAllocation{
				DataPath:  dataPath,
				MetaPath:  metaPath,
				Capacity:  1,
				S3APIPort: 4900,
				RPCPort:   4901,
				AdminPort: 4902,
			},
		)

		// Creating the directory is enough to ensure that Isle chooses LMDB as
		// the db engine.
		lmdbPath := filepath.Join(metaPath, "db.lmdb")
		require.NoError(t, os.Mkdir(lmdbPath, 0755))

		assert.NoError(t, network.SetConfig(h.ctx, networkConfig))

		t.Log("Checking that garage is using the expected db engine")
		garageConfig, err := os.ReadFile(
			filepath.Join(network.runtimeDir.Path, "garage-4901.toml"),
		)
		assert.NoError(t, err)
		assert.Contains(t,
			string(garageConfig),
			`db_engine = "`+garagesrv.DBEngineLMDB+`"`,
		)
		assert.NoFileExists(t, filepath.Join(metaPath, "db.sqlite"))
	})

	t.Run("remove storage alloc", func(t *testing.T) {
		var (
			h       = newIntegrationHarness(t)
			network = h.createNetwork(t, "primus", &createNetworkOpts{
				numStorageAllocs: 4,
			})
			networkConfig = network.getConfig(t)

			prevHost          = network.getHostsByName(t)[network.hostName]
			removedAlloc      = networkConfig.Storage.Allocations[3]
			removedGarageInst = daecommon.BootstrapGarageHostForAlloc(
				prevHost, removedAlloc,
			)
		)

		networkConfig.Storage.Allocations = networkConfig.Storage.Allocations[:3]
		assert.NoError(t, network.SetConfig(h.ctx, networkConfig))

		t.Log("Checking that the Host information was updated")
		newHostsByName := network.getHostsByName(t)
		newHost, ok := newHostsByName[network.hostName]
		assert.True(t, ok)

		allocs := newHost.HostConfigured.Garage.Instances
		assert.Len(t, allocs, 3)
		assert.NotContains(t, allocs, removedGarageInst)

		t.Log("Checking that the bootstrap file was written with the new host config")
		var storedBootstrap bootstrap.Bootstrap
		assert.NoError(t, jsonutil.LoadFile(
			&storedBootstrap, bootstrap.StateDirPath(network.stateDir.Path),
		))
		assert.Equal(t, newHostsByName, storedBootstrap.Hosts)

		t.Log("Checking that garage layout contains the new allocation")
		expRoles := allocsToRoles(network.hostName, allocs)
		layout, err := network.garageAdminClient(t).GetLayout(h.ctx)
		assert.NoError(t, err)
		assert.ElementsMatch(t, expRoles, layout.Roles)
	})

	t.Run("remove all storage allocs", func(t *testing.T) {
		t.Skip("This is currently expected to fail. Orphan removal is going to be reworked accordingly")

		var (
			h        = newIntegrationHarness(t)
			primus   = h.createNetwork(t, "primus", nil)
			secondus = h.joinNetwork(t, primus, "secondus", &joinNetworkOpts{
				networkConfigOpts: &networkConfigOpts{
					numStorageAllocs: 1,
				},
			})
			networkConfig = secondus.getConfig(t)

			prevHost    = secondus.getHostsByName(t)[secondus.hostName]
			removedRole = allocsToRoles(
				secondus.hostName, prevHost.Garage.Instances,
			)[0]

			primusGarageAdminClient = primus.garageAdminClient(t)
		)

		networkConfig.Storage.Allocations = nil
		assert.NoError(t, secondus.SetConfig(h.ctx, networkConfig))

		t.Log("Checking that the Host information was updated")
		newHostsByName := primus.getHostsByName(t)
		newHost, ok := newHostsByName[secondus.hostName]
		assert.True(t, ok)

		allocs := newHost.HostConfigured.Garage.Instances
		assert.Empty(t, allocs)

		t.Log("Checking that garage layout still contains the old allocation")
		layout, err := primusGarageAdminClient.GetLayout(h.ctx)
		assert.NoError(t, err)
		assert.Contains(t, layout.Roles, removedRole)

		//t.Log("Removing orphan garage nodes with primus")
		//assert.NoError(
		//	t, primus.Network.(*network).removeOrphanGarageNodes(h.ctx),
		//)

		t.Log("Checking that garage layout no longer contains the old allocation")
		layout, err = primusGarageAdminClient.GetLayout(h.ctx)
		assert.NoError(t, err)
		assert.NotContains(t, layout.Roles, removedRole)
	})

	t.Run("changes reflected after restart", func(t *testing.T) {
		var (
			h       = newIntegrationHarness(t)
			network = h.createNetwork(t, "primus", &createNetworkOpts{
				numStorageAllocs: 4,
			})
			networkConfig = network.getConfig(t)
		)

		networkConfig.Storage.Allocations = networkConfig.Storage.Allocations[:3]
		assert.NoError(t, network.SetConfig(h.ctx, networkConfig))

		network.opts.Config = nil
		network.restart(t)

		assert.Equal(t, networkConfig, network.getConfig(t))
	})
}