Refactor tests some more, use shared across all of them

go/cmd/entrypoint/bootstrap_util.go

@@ -1,10 +1,10 @@
 package main
 
 import (
-	"isle/bootstrap"
 	"errors"
 	"fmt"
 	"io/fs"
+	"isle/bootstrap"
 	"os"
 	"path/filepath"
 )
@@ -15,7 +15,10 @@ func loadHostBootstrap() (bootstrap.Bootstrap, error) {
 
 	hostBootstrap, err := bootstrap.FromFile(dataDirPath)
 	if errors.Is(err, fs.ErrNotExist) {
-		return bootstrap.Bootstrap{}, errors.New("%q not found, has the daemon ever been run?")
+		return bootstrap.Bootstrap{}, fmt.Errorf(
+			"%q not found, has the daemon ever been run?",
+			dataDirPath,
+		)
 
 	} else if err != nil {
 		return bootstrap.Bootstrap{}, fmt.Errorf("loading %q: %w", dataDirPath, err)

tests/cases/admin/01-create-network.sh

@@ -1,34 +1,5 @@
-# shellcheck source=../utils/with-tmp-for-case.sh
-source "$UTILS"/with-tmp-for-case.sh
-
-mkdir a
-mkdir b
-mkdir c
-
-cat >daemon.yml <<EOF
-vpn:
-  tun:
-    device: isle-test
-storage:
-  allocations:
-    - data_path: a/data
-      meta_path: a/meta
-      capacity: 100
-    - data_path: b/data
-      meta_path: b/meta
-      capacity: 100
-    - data_path: c/data
-      meta_path: c/meta
-      capacity: 100
-EOF
-
-isle admin create-network \
-    --config-path daemon.yml \
-    --domain      test.isle.com \
-    --hostname    primus \
-    --ip-net      "10.6.9.1/24" \
-    --name        "testing" \
-    > admin.yml
+# shellcheck source=../../utils/with-1-data-1-empty-node-cluster.sh
+source "$UTILS"/with-1-data-1-empty-node-cluster.sh
 
 [ "$(cat a/meta/isle/rpc_port)" = "3900" ]
 [ "$(cat b/meta/isle/rpc_port)" = "3910" ]
@@ -36,7 +7,7 @@ isle admin create-network \
 
 [ "$(yq <admin.yml '.creation_params.id')" != "" ]
 [ "$(yq <admin.yml '.creation_params.name')" = "testing" ]
-[ "$(yq <admin.yml '.creation_params.domain')" = "test.isle.com" ]
+[ "$(yq <admin.yml '.creation_params.domain')" = "shared.test" ]
 
 bootstrap_file="$XDG_DATA_HOME/isle/bootstrap.yml"
 

tests/cases/admin/02-create-bootstrap.sh

@@ -1,14 +1,8 @@
-# shellcheck source=../utils/with-single-node-cluster.sh
-source "$UTILS"/with-single-node-cluster.sh
+# shellcheck source=../../utils/with-1-data-1-empty-node-cluster.sh
+source "$UTILS"/with-1-data-1-empty-node-cluster.sh
 
 adminBS="$XDG_DATA_HOME"/isle/bootstrap.yml
-bs=create-bootstrap-out.yml
-
-isle admin create-bootstrap \
-    --admin-path admin.yml \
-    --hostname secondus \
-    --ip 10.6.9.2 \
-    > "$bs"
+bs="$secondus_bootstrap" # set in with-1-data-1-empty-node-cluster.sh
 
 [ "$(yq <"$bs" '.admin_creation_params')" = "$(yq <admin.yml '.creation_params')" ]
 [ "$(yq <"$bs" '.hostname')" = "secondus" ]

tests/cases/garage/00-cli.sh

@@ -1,5 +1,5 @@
-# shellcheck source=../../utils/with-single-node-cluster.sh
-source "$UTILS"/with-single-node-cluster.sh
+# shellcheck source=../../utils/with-1-data-1-empty-node-cluster.sh
+source "$UTILS"/with-1-data-1-empty-node-cluster.sh
 
 status="$(isle garage cli status | tail -n+3)"
 

tests/cases/garage/01-mc.sh

@@ -1,5 +1,5 @@
-# shellcheck source=../../utils/with-single-node-cluster.sh
-source "$UTILS"/with-single-node-cluster.sh
+# shellcheck source=../../utils/with-1-data-1-empty-node-cluster.sh
+source "$UTILS"/with-1-data-1-empty-node-cluster.sh
 
 files="$(isle garage mc -- tree --json garage)"
 [ "$(echo "$files" | jq -s '.|length')" -ge "1" ]

tests/entrypoint.sh

@@ -35,11 +35,15 @@ EOF
   esac
 done
 
-TMPDIR="$(mktemp --tmpdir -d isle-tests.XXXXXX)"
-if [ -z "$KEEP_TMP" ]; then trap 'rm -rf $TMPDIR' EXIT; fi
+[ -n "$VERBOSE" ] && set -x
 
-export TMPDIR
-echo "tmp dir is $TMPDIR"
+ROOT_TMPDIR="$(mktemp --tmpdir -d isle-tests.XXXXXX)"
+if [ -z "$KEEP_TMP" ]; then trap 'rm -rf $ROOT_TMPDIR' EXIT; fi
+
+TMPDIR="$ROOT_TMPDIR"
+
+export ROOT_TMPDIR TMPDIR
+echo "tmp dir is $ROOT_TMPDIR"
 
 # Blackhole these directories so that tests don't accidentally use the host's
 # real ones.
@@ -89,9 +93,22 @@ done
 
 # Clean up any shared running clusters. Each cleanup script is responsible for
 # figuring out if its shared cluster was actually instantiated during any tests.
-echo "Running any cleanup tasks"
 
-# shellcheck source=./utils/cleanup-single-node-cluster.sh
-source "$UTILS"/cleanup-single-node-cluster.sh
+if [ -e "$ROOT_TMPDIR/cleanup-pids" ]; then
+    echo "Cleaning up running pids"
+    tac "$ROOT_TMPDIR/cleanup-pids" | while read -r line; do
+        pid="$(echo "$line" | cut -d' ' -f1)"
+        descr="$(echo "$line" | cut -d' ' -f2-)"
+        echo "Killing $descr ($pid)"
+        kill "$pid"
+    done
+
+    # This is easier than checking if the pids are still running, and for some
+    # reason it doesn't occur until after the pids have died anyway
+    echo "Waiting for appimage mounts to unmount"
+    while [ "$(find "$ROOT_TMPDIR" -type d -name '*.mount_isle*' | wc -l)" -ge "1" ]; do
+        sleep 1
+    done
+fi
 
 if [ -z "$TESTS_FAILED" ]; then echo -e '\nall tests succeeded!'; fi

tests/utils/cleanup-single-node-cluster.sh

@@ -1,14 +0,0 @@
-(
-    set -e
-
-    TMPDIR="$TMPDIR/shared/single-node.tmp"
-    if [ ! -d "$TMPDIR" ]; then exit 0; fi
-
-    lock_file="$TMPDIR/.run/isle/lock"
-    if [ ! -e "$lock_file" ]; then exit 0; fi
-
-    pid="$(cat "$lock_file")"
-    echo "killing shared single node cluster (process: $pid)"
-    kill "$pid"
-    while [ -e "$TMPDIR/.run/isle" ]; do sleep 1; done
-)

tests/utils/register-cleanup.sh

@@ -0,0 +1,3 @@
+set -e
+
+echo "$1" "$2" >> "$ROOT_TMPDIR/cleanup-pids"

tests/utils/shared-daemon-env.sh

@@ -0,0 +1,11 @@
+set -e
+
+base="$1"
+
+TMPDIR="$ROOT_TMPDIR/$base"
+XDG_RUNTIME_DIR="$TMPDIR/.run"
+XDG_DATA_HOME="$TMPDIR/.data"
+
+mkdir -p "$TMPDIR" "$XDG_RUNTIME_DIR" "$XDG_DATA_HOME"
+
+echo "export TMPDIR='$TMPDIR' XDG_RUNTIME_DIR='$XDG_RUNTIME_DIR' XDG_DATA_HOME='$XDG_DATA_HOME'"

tests/utils/with-1-data-1-empty-node-cluster.sh

@@ -0,0 +1,81 @@
+set -e
+
+base="shared/1-data-1-empty"
+primus_base="$base/primus"
+secondus_base="$base/secondus"
+
+# Even if it's already intialized, we want to put the caller in primus'
+# environment
+eval "$($SHELL "$UTILS/shared-daemon-env.sh" "$primus_base")"
+cd "$TMPDIR"
+
+secondus_bootstrap="$(pwd)/secondus-bootstrap.yml"
+
+if [ ! -d "$XDG_RUNTIME_DIR/isle" ]; then
+    echo "Initializing shared single node cluster"
+
+    mkdir a
+    mkdir b
+    mkdir c
+
+    cat >daemon.yml <<EOF
+    vpn:
+      public_addr: 127.0.0.1:60000
+      tun:
+        device: isle-primus
+    storage:
+      allocations:
+        - data_path: a/data
+          meta_path: a/meta
+          capacity: 100
+        - data_path: b/data
+          meta_path: b/meta
+          capacity: 100
+        - data_path: c/data
+          meta_path: c/meta
+          capacity: 100
+EOF
+
+    echo "Creating 1-data-1-empty network"
+    isle admin create-network \
+        --config-path daemon.yml \
+        --domain      shared.test \
+        --hostname    primus \
+        --ip-net      "10.6.9.1/24" \
+        --name        "testing" \
+        > admin.yml
+
+    isle daemon --config-path daemon.yml >daemon.log 2>&1 &
+    pid="$!"
+    echo "Waiting for primus daemon (process $pid) to initialize"
+
+    while ! isle hosts list >/dev/null; do sleep 1; done
+
+    $SHELL "$UTILS/register-cleanup.sh" "$pid" "1-data-1-empty-node-cluster/primus"
+
+    echo "Creating secondus bootstrap"
+    isle admin create-bootstrap \
+        --admin-path admin.yml \
+        --hostname secondus \
+        --ip 10.6.9.2 \
+        > "$secondus_bootstrap"
+
+    (
+        eval "$($SHELL "$UTILS/shared-daemon-env.sh" "$secondus_base")"
+        cd "$TMPDIR"
+
+        cat >daemon.yml <<EOF
+        vpn:
+          tun:
+            device: isle-secondus
+EOF
+
+        isle daemon -c daemon.yml -b "$secondus_bootstrap" >daemon.log 2>&1 &
+        pid="$!"
+        echo "Waiting for secondus daemon (process $!) to initialize"
+
+        while ! isle hosts list >/dev/null; do sleep 1; done
+
+        $SHELL "$UTILS/register-cleanup.sh" "$pid" "1-data-1-empty-node-cluster/secondus"
+    )
+fi

tests/utils/with-single-node-cluster.sh

@@ -1,48 +0,0 @@
-set -e
-
-TMPDIR="$TMPDIR/shared/single-node.tmp"
-XDG_RUNTIME_DIR="$TMPDIR/.run"
-XDG_DATA_HOME="$TMPDIR/.data"
-
-mkdir -p "$TMPDIR" "$XDG_RUNTIME_DIR" "$XDG_DATA_HOME"
-
-cd "$TMPDIR"
-
-if [ ! -d "$XDG_RUNTIME_DIR/isle" ]; then
-    echo "Initializing shared single node cluster"
-
-    mkdir a
-    mkdir b
-    mkdir c
-
-    cat >daemon.yml <<EOF
-    vpn:
-      public_addr: 127.0.0.1:60000
-      tun:
-        device: isle-test-shared
-    storage:
-      allocations:
-        - data_path: a/data
-          meta_path: a/meta
-          capacity: 100
-        - data_path: b/data
-          meta_path: b/meta
-          capacity: 100
-        - data_path: c/data
-          meta_path: c/meta
-          capacity: 100
-EOF
-
-    isle admin create-network \
-        --config-path daemon.yml \
-        --domain      shared.test \
-        --hostname    primus \
-        --ip-net      "10.6.9.1/24" \
-        --name        "testing" \
-        > admin.yml
-
-    isle daemon --config-path daemon.yml >daemon.log 2>&1 &
-    echo "Waiting for daemon (process $!) to initialize"
-
-    while ! isle hosts list >/dev/null; do sleep 1; done
-fi