Remove legacy tests, document new ones

default.nix

@@ -118,19 +118,6 @@ in rec {
     '';
   };
 
-  tests = pkgs.writeScript "isle-tests" ''
-    export PATH=${pkgs.lib.makeBinPath [
-      build.appImage
-      pkgs.busybox
-      pkgs.yq-go
-      pkgs.jq
-      pkgs.dig
-      pkgs.nebula
-    ]}
-    export SHELL=${pkgs.bash}/bin/bash
-    exec ${pkgs.bash}/bin/bash ${./tests}/entrypoint.sh "$@"
-  '';
-
   devShell = pkgs.mkShell {
     buildInputs = [
       pkgs.go

docs/dev/index.md

@@ -11,5 +11,6 @@ to better understand how to navigate and work on the codebase.
 These pages can be helpful in specific situations.
 
 * [Building Isle](./building.md)
+* [Testing Isle](./testing.md)
 * [Rebuilding Documentation](./rebuilding-documentation.md)
 * [Releases](./releases.md)

docs/dev/testing.md

@@ -0,0 +1,39 @@
+# Testing Isle
+
+All tests are currently written as go tests, and as such can be run from the
+`go` directory using the normal go testing tool.
+
+```
+cd go
+go test -run Foo ./daemon
+go test ./... # Test everything
+```
+
+## Integration Tests
+
+Integration tests are those which require processes or state external to the
+test itself. Integration tests are marked using the
+`toolkit.MarkIntegrationTest` function, which will cause them to be skipped
+unless being run in the integration test environment.
+
+Besides a normal nix installation (like all Isle development needs), integration
+tests also require `sudo` and [capsh][capsh] to be installed on the system.
+
+[capsh]: https://www.man7.org/linux/man-pages/man1/capsh.1.html
+
+By running tests using the `go/integration_test.sh` script the tests will be
+automatically run in the integration test environment. All arguments will be
+passed directly to the go testing tool.
+
+```
+cd go
+./integration_test.sh -run Foo ./daemon
+```
+
+`integration_test.sh` wraps a call to `go test` in a bash shell which has all
+required binaries available to it, and which has acquired necessary
+[capabilities][capabilities] to use the binaries as needed. Acquiring
+capabilities is done by elevating the user to root using `sudo`, and then
+dropping them back down to a shell of the original user with capabilities set.
+
+[capabilities]: https://wiki.archlinux.org/title/Capabilities

tests.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-entrypoint="$(nix-build --no-out-link -A tests)"
-this_user="$(whoami)"
-
-echo "Requesting sudo in order to set thread capabilities, will drop back down to user '$this_user' immediately"
-
-sudo -E capsh \
-    --caps="cap_net_admin,cap_net_bind_service+eip cap_setpcap,cap_setuid,cap_setgid+ep" \
-    --keep=1 \
-    --user="$this_user" \
-    --addamb=cap_net_admin \
-    --addamb=cap_net_bind_service \
-    -- "$entrypoint" "$@"

tests/NOTES.txt

@@ -1,10 +0,0 @@
-Ctrl+A X -> exits
-
-qemu-system-aarch64 -nographic -cdrom tests/alpine-virt-3.18.4-aarch64.iso
-
-Ctrl+Alt+G -> Escape mouse capture
-qemu-system-x86_64 \
-    -cdrom tests/virt/Win11_23H2_English_x64.iso \
-    -m 8G \
-    -boot order=d \
-    -drive file=./tests/virt/winblows.qcow2

tests/cases/00-version.sh

@@ -1,3 +0,0 @@
-isle version | grep -q 'Release:'
-isle version | grep -q 'Platform:'
-isle version | grep -q 'Build Platform:'

tests/cases/dnsmasq/00-hosts.sh

@@ -1,17 +0,0 @@
-# shellcheck source=../../utils/with-1-data-1-empty-node-network.sh
-source "$UTILS"/with-1-data-1-empty-node-network.sh
-
-function assert_a {
-    want_ip="$1"
-    hostname="$2"
-    r="$(dig @"$current_ip" +noall +answer "$hostname")"
-    echo "$r" | grep -q "$want_ip"
-}
-
-as_primus
-assert_a "$primus_ip" primus.hosts.shared.test
-assert_a "$secondus_ip" secondus.hosts.shared.test
-
-as_secondus
-assert_a "$primus_ip" primus.hosts.shared.test
-assert_a "$secondus_ip" secondus.hosts.shared.test

tests/cases/garage/00-cli.sh

@@ -1,21 +0,0 @@
-# shellcheck source=../../utils/with-1-data-1-empty-node-network.sh
-source "$UTILS"/with-1-data-1-empty-node-network.sh
-
-function do_tests {
-    status="$(isle garage cli status | tail -n+3)"
-
-    [ "$(echo "$status" | wc -l)" = "3" ]
-    echo "$status" | grep -q '10.6.9.1:3900'
-    echo "$status" | grep -q '10.6.9.1:3910'
-    echo "$status" | grep -q '10.6.9.1:3920'
-
-    buckets="$(isle garage cli bucket list | tail -n+2)"
-    [ "$(echo "$buckets" | wc -l)" = 1 ]
-    echo "$buckets" | grep -q 'global-shared'
-}
-
-as_primus
-do_tests
-
-as_secondus
-do_tests

tests/cases/garage/01-mc.sh

@@ -1,16 +0,0 @@
-# shellcheck source=../../utils/with-1-data-1-empty-node-network.sh
-source "$UTILS"/with-1-data-1-empty-node-network.sh
-
-function do_tests {
-    files="$(isle garage mc -- tree --json garage)"
-    [ "$(echo "$files" | jq -s '.|length')" -ge "1" ]
-
-    file="$(echo "$files" | jq -sr '.[0].key')"
-    [ "$(isle garage mc -- cat "garage/$file" | wc -c)" -gt "0" ]
-}
-
-as_primus
-do_tests
-
-as_secondus
-do_tests

tests/cases/hosts/00-list.sh

@@ -1,20 +0,0 @@
-# shellcheck source=../../utils/with-1-data-1-empty-node-network.sh
-source "$UTILS"/with-1-data-1-empty-node-network.sh
-
-function do_tests {
-    hosts="$(isle hosts list)"
-
-    [ "$(echo "$hosts" | jq -r '.[0].Name')" = "primus" ]
-    [ "$(echo "$hosts" | jq -r '.[0].VPN.IP')" = "10.6.9.1" ]
-    [ "$(echo "$hosts" | jq -r '.[0].Storage.Instances|length')" = "3" ]
-
-    [ "$(echo "$hosts" | jq -r '.[1].Name')" = "secondus" ]
-    [ "$(echo "$hosts" | jq -r '.[1].VPN.IP')" = "$secondus_ip" ]
-    [ "$(echo "$hosts" | jq -r '.[1].Storage.Instances|length')" = "0" ]
-}
-
-as_primus
-do_tests
-
-as_secondus
-do_tests

tests/cases/hosts/01-create.sh

@@ -1,16 +0,0 @@
-# shellcheck source=../../utils/with-1-data-1-empty-node-network.sh
-source "$UTILS"/with-1-data-1-empty-node-network.sh
-
-bs="$secondus_bootstrap" # set in with-1-data-1-empty-node-network.sh
-
-[ "$(jq -r <"$bs" '.Bootstrap.NetworkCreationParams.Domain')" = "shared.test" ]
-[ "$(jq -r <"$bs" '.Bootstrap.NetworkCreationParams.Name')" = "testing" ]
-[ "$(jq -r <"$bs" '.Bootstrap.SignedHostAssigned.Body.Name')" = "secondus" ]
-
-[ "$(jq -r <"$bs" '.Bootstrap.Hosts.primus.PublicCredentials')" \
-= "$(jq -r <"$BOOTSTRAP_FILE" '.SignedHostAssigned.Body.PublicCredentials')" ]
-
-[ "$(jq <"$bs" '.Bootstrap.Hosts.primus.Garage.Instances|length')" = "3" ]
-
-[ "$(jq <"$bs" '.Secrets["garage-rpc-secret"]')" != "null" ]
-

tests/cases/nebula/00-show.sh

@@ -1,12 +0,0 @@
-# shellcheck source=../../utils/with-1-data-1-empty-node-network.sh
-source "$UTILS"/with-1-data-1-empty-node-network.sh
-
-info="$(isle nebula show)"
-
-[ "$(echo "$info" | jq -r '.CACert')" \
-= "$(jq -r <"$BOOTSTRAP_FILE" '.CAPublicCredentials.Cert')" ]
-
-[ "$(echo "$info" | jq -r '.SubnetCIDR')" = "10.6.9.0/24" ]
-[ "$(echo "$info" | jq -r '.Lighthouses|length')" = "1" ]
-[ "$(echo "$info" | jq -r '.Lighthouses[0].PublicAddr')" = "127.0.0.1:60000" ]
-[ "$(echo "$info" | jq -r '.Lighthouses[0].IP')" = "10.6.9.1" ]

tests/cases/nebula/01-create-cert.sh

@@ -1,17 +0,0 @@
-# shellcheck source=../../utils/with-1-data-1-empty-node-network.sh
-source "$UTILS"/with-1-data-1-empty-node-network.sh
-
-nebula-cert keygen -out-key /dev/null -out-pub pubkey
-cat pubkey
-
-(
-    isle nebula create-cert \
-        --hostname non-esiste \
-        --public-key-path pubkey \
-        2>&1 || true \
-) | grep '\[1002\] Host not found'
-
-isle nebula create-cert \
-    --hostname primus \
-    --public-key-path pubkey \
-    | grep -- '-----BEGIN NEBULA CERTIFICATE-----'

tests/cases/network/00-create.sh

@@ -1,12 +0,0 @@
-# shellcheck source=../../utils/with-1-data-1-empty-node-network.sh
-source "$UTILS"/with-1-data-1-empty-node-network.sh
-
-[ "$(cat a/meta/isle/rpc_port)" = "3900" ]
-[ "$(cat b/meta/isle/rpc_port)" = "3910" ]
-[ "$(cat c/meta/isle/rpc_port)" = "3920" ]
-
-[ "$(jq -r <"$BOOTSTRAP_FILE" '.NetworkCreationParams.ID')" != "" ]
-[ "$(jq -r <"$BOOTSTRAP_FILE" '.NetworkCreationParams.Name')" = "testing" ]
-[ "$(jq -r <"$BOOTSTRAP_FILE" '.NetworkCreationParams.Domain')" = "shared.test" ]
-
-[ "$(jq -r <"$BOOTSTRAP_FILE" '.SignedHostAssigned.Body.Name')" = "primus" ]

tests/entrypoint.sh

@@ -1,115 +0,0 @@
-set -e
-
-# cd into script's directory
-cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null
-root=$(pwd)
-
-export UTILS="$root"/utils
-
-REGEXS=()
-
-while [[ $# -gt 0 ]]; do
-  case $1 in
-    -h|--help)
-        cat <<EOF
-USAGE: [flags] [test regexs...]
-FLAGS
-    --keep-tmp
-    --verbose (-v)
-    --help    (-h)
-EOF
-      exit 1
-      ;;
-    -v|--verbose)
-      VERBOSE=1
-      shift
-      ;;
-    --keep-tmp)
-      KEEP_TMP=1
-      shift
-      ;;
-    *)
-      REGEXS+=("$1")
-      shift
-      ;;
-  esac
-done
-
-[ -n "$VERBOSE" ] && set -x
-
-ROOT_TMPDIR="$(mktemp --tmpdir -d isle-tests.XXXXXX)"
-if [ -z "$KEEP_TMP" ]; then trap 'rm -rf $ROOT_TMPDIR' EXIT; fi
-
-TMPDIR="$ROOT_TMPDIR"
-
-export ROOT_TMPDIR TMPDIR
-echo "tmp dir is $ROOT_TMPDIR"
-
-# Blackhole these directories so that tests don't accidentally use the host's
-# real ones.
-export XDG_RUNTIME_DIR=/dev/null
-export XDG_STATE_HOME=/dev/null
-
-test_files=$(
-    find ./cases -type f -name '*.sh' \
-    | sed "s|^\./cases/||" \
-    | grep -v entrypoint.sh \
-    | sort
-)
-
-for r in "${REGEXS[@]}"; do
-    test_files="$(echo "$test_files" | grep "$r")"
-done
-
-echo -e "number of tests: $(echo "$test_files" | wc -l)\n"
-for file in $test_files; do
-    echo "Running test case: $file"
-
-    if [ -z "$VERBOSE" ]; then
-        output="$TMPDIR/$file.log"
-        mkdir -p "$(dirname "$output")"
-        exec 3>"$output"
-    else
-        exec 3>&1
-    fi
-
-    (
-        export TEST_CASE_FILE="$file"
-
-        if ! $SHELL -e -x "$root/cases/$file" >&3 2>&1; then
-            echo "$file FAILED"
-            if [ -z "$VERBOSE" ]; then
-                echo "output of test is as follows"
-                echo "------------------------------"
-                cat "$output"
-                echo "------------------------------"
-            fi
-            exit 1
-        fi
-    ) || TESTS_FAILED=1
-
-    if [ -n "$TESTS_FAILED" ]; then break; fi
-done
-
-
-# Clean up any shared running networks. Each cleanup script is responsible for
-# figuring out if its shared network was actually instantiated during any tests.
-
-if [ -e "$ROOT_TMPDIR/cleanup-pids" ]; then
-    echo "Cleaning up running pids"
-    tac "$ROOT_TMPDIR/cleanup-pids" | while read -r line; do
-        pid="$(echo "$line" | cut -d' ' -f1)"
-        descr="$(echo "$line" | cut -d' ' -f2-)"
-        echo "Killing $descr ($pid)"
-        kill "$pid"
-    done
-
-    # This is easier than checking if the pids are still running, and for some
-    # reason it doesn't occur until after the pids have died anyway
-    echo "Waiting for appimage mounts to unmount"
-    while [ "$(find "$ROOT_TMPDIR" -type d -name '*.mount_isle*' | wc -l)" -ge "1" ]; do
-        sleep 1
-    done
-fi
-
-if [ -z "$TESTS_FAILED" ]; then echo -e '\nall tests succeeded!'; fi

tests/utils/register-cleanup.sh

@@ -1,3 +0,0 @@
-set -e
-
-echo "$1" "$2" >> "$ROOT_TMPDIR/cleanup-pids"

tests/utils/shared-daemon-env.sh

@@ -1,18 +0,0 @@
-set -e
-
-base="$1"
-
-TMPDIR="$ROOT_TMPDIR/$base"
-XDG_RUNTIME_DIR="$TMPDIR/.run"
-XDG_STATE_HOME="$TMPDIR/.state"
-
-mkdir -p "$TMPDIR" "$XDG_RUNTIME_DIR" "$XDG_STATE_HOME"
-
-cat <<EOF
-export TMPDIR="$TMPDIR"
-export XDG_RUNTIME_DIR="$XDG_RUNTIME_DIR"
-export XDG_STATE_HOME="$XDG_STATE_HOME"
-export ISLE_DAEMON_HTTP_SOCKET_PATH="$ROOT_TMPDIR/$base-daemon.sock"
-BOOTSTRAP_FILE="$XDG_STATE_HOME/isle/networks/$NETWORK_ID/bootstrap.json"
-cd "$TMPDIR"
-EOF

tests/utils/with-1-data-1-empty-node-network.sh

@@ -1,107 +0,0 @@
-set -e
-
-base="shared/1-data-1-empty"
-
-ipNet="10.6.9.0/24"
-
-primus_base="$base/primus"
-primus_ip="10.6.9.1"
-
-secondus_base="$base/secondus"
-
-function as_primus {
-    current_ip="$primus_ip"
-    eval "$($SHELL "$UTILS/shared-daemon-env.sh" "$primus_base")"
-}
-
-function as_secondus {
-    current_ip="$secondus_ip"
-    eval "$($SHELL "$UTILS/shared-daemon-env.sh" "$secondus_base")"
-}
-
-# Even if it's already intialized, we want to put the caller in primus'
-# environment
-as_primus
-
-secondus_bootstrap="$(pwd)/secondus-bootstrap.json"
-
-if [ ! -d "$XDG_RUNTIME_DIR/isle" ]; then
-    echo "Initializing shared single node network"
-
-    mkdir a
-    mkdir b
-    mkdir c
-
-    cat >daemon.yml <<EOF
-    networks:
-      testing:
-        vpn:
-          public_addr: 127.0.0.1:60000
-          tun:
-            device: isle-primus
-        storage:
-          allocations:
-            - data_path: a/data
-              meta_path: a/meta
-              capacity: 1
-            - data_path: b/data
-              meta_path: b/meta
-              capacity: 1
-            - data_path: c/data
-              meta_path: c/meta
-              capacity: 1
-EOF
-
-    isle daemon -l debug --config-path daemon.yml >daemon.log 2>&1 &
-    pid="$!"
-    $SHELL "$UTILS/register-cleanup.sh" "$pid" "1-data-1-empty-node-network/primus"
-
-    echo "Waiting for primus daemon (process $pid) to start"
-    while ! [ -e "$ISLE_DAEMON_HTTP_SOCKET_PATH" ]; do sleep 1; done
-
-    echo "Creating 1-data-1-empty network"
-    isle network create \
-        --domain      shared.test \
-        --hostname    primus \
-        --ip-net      "$ipNet" \
-        --name        "testing"
-
-    echo "Creating secondus bootstrap"
-    isle hosts create \
-        --hostname secondus \
-        > "$secondus_bootstrap"
-
-    (
-        as_secondus
-
-        cat >daemon.yml <<EOF
-        vpn:
-          tun:
-            device: isle-secondus
-EOF
-
-        isle daemon -l debug -c daemon.yml >daemon.log 2>&1 &
-        pid="$!"
-        $SHELL "$UTILS/register-cleanup.sh" "$pid" "1-data-1-empty-node-network/secondus"
-
-        echo "Waiting for secondus daemon (process $!) to start"
-        while ! [ -e "$ISLE_DAEMON_HTTP_SOCKET_PATH" ]; do sleep 1; done
-
-        echo "Joining secondus to the network"
-        isle network join -b "$secondus_bootstrap"
-    )
-fi
-
-secondus_ip="$(
-    nebula-cert print -json \
-        -path <(jq -r '.Bootstrap.Hosts["secondus"].PublicCredentials.Cert' "$secondus_bootstrap") \
-    | jq -r '.details.ips[0]' \
-    | cut -d/ -f1
-)"
-
-NETWORK_ID="$(jq '.Bootstrap.NetworkCreationParams.ID' "$secondus_bootstrap")"
-export NETWORK_ID
-
-# shared-daemon-env.sh depends on NETWORK_ID, so we re-call as_primus in order
-# to fully populate the envvars we need.
-as_primus

tests/utils/with-tmp-for-case.sh

@@ -1,9 +0,0 @@
-set -e
-
-TMPDIR="$TMPDIR/$TEST_CASE_FILE.tmp"
-XDG_RUNTIME_DIR="$TMPDIR/.run"
-XDG_STATE_HOME="$TMPDIR/.state"
-
-mkdir -p "$TMPDIR" "$XDG_RUNTIME_DIR" "$XDG_STATE_HOME"
-
-cd "$TMPDIR"