From ee7c739304098de1eb7faa7fb82974261748badf Mon Sep 17 00:00:00 2001 From: Luke Hinds Date: Fri, 15 May 2026 12:14:59 +0100 Subject: [PATCH] test(integration): fix linux int test suite robustness and isolation This commit introduces several refinements to the integration test suite to enhance its reliability, isolation, and accuracy across various scenarios. - Improve test isolation by ensuring `HOME` and `XDG_CONFIG_HOME` are explicitly set for relevant profile and client startup tests. - Robustly handle optional pack installations (e.g., `always-further/claude`) in client startup tests, including verification logic to prevent failures in CI environments. - Correct various test assertions to align with expected behavior and system limitations: - Assert audit session creation for read-only paths, as these do not create rollback snapshots. - Update the expected behavior for broad `/sys` directory access, which should not be granted by default. - Clarify that Landlock's `--block-net` does not filter ICMP traffic, updating the ping test to skip instead of fail. - Update expected warning messages for skipped non-existent file grants. - Refine expectations for default and verbose dry-run output, focusing on capability provenance for optional profile paths. - Enhance the build setup for test execution by making the determination of the target release directory more robust. Signed-off-by: Luke Hinds --- tests/integration/test_audit.sh | 6 ++- tests/integration/test_bypass_protection.sh | 13 ++++--- tests/integration/test_client_startup.sh | 43 ++++++++++++++++----- tests/integration/test_edge_cases.sh | 2 +- tests/integration/test_network.sh | 8 ++-- tests/integration/test_profiles.sh | 3 ++ tests/integration/test_silent_output.sh | 16 ++++---- tests/integration/test_system_paths.sh | 2 +- tests/run_integration_tests.sh | 8 +++- 9 files changed, 68 insertions(+), 33 deletions(-) diff --git a/tests/integration/test_audit.sh b/tests/integration/test_audit.sh index 6f45bbe42..f548d7647 100755 --- a/tests/integration/test_audit.sh +++ b/tests/integration/test_audit.sh @@ -133,10 +133,12 @@ expect_failure "--no-audit --rollback is rejected" \ echo "" echo "--- Audit with Rollback ---" -# Test 5: --rollback with read-only paths still creates an audit session +# Test 5: --rollback with read-only paths still creates an audit session. +# Read-only grants do not create rollback snapshots because there are no +# writable paths to track, so assert on the audit root rather than rollback root. TESTS_RUN=$((TESTS_RUN + 1)) run_nono run --silent --rollback --no-rollback-prompt --allow-cwd --read "$TMPDIR" -- echo "readonly rollback audit" -session_file=$(find_rollback_session_for_pid "$LAST_NONO_PID") +session_file=$(find_audit_session_for_pid "$LAST_NONO_PID") if [[ -n "$session_file" && -f "$session_file" ]]; then echo -e " ${GREEN}PASS${NC}: rollback read-only session creates audit" TESTS_PASSED=$((TESTS_PASSED + 1)) diff --git a/tests/integration/test_bypass_protection.sh b/tests/integration/test_bypass_protection.sh index d1cb6259a..10525f5ee 100755 --- a/tests/integration/test_bypass_protection.sh +++ b/tests/integration/test_bypass_protection.sh @@ -22,8 +22,14 @@ fi TMPDIR=$(setup_test_dir) trap 'cleanup_test_dir "$TMPDIR"' EXIT +TEST_HOME="$TMPDIR/home" +export HOME="$TEST_HOME" +export XDG_CONFIG_HOME="$TEST_HOME/.config" + PROFILES_DIR="$TMPDIR/profiles" -mkdir -p "$PROFILES_DIR" +mkdir -p "$PROFILES_DIR" "$HOME/.docker" "$HOME/.ssh" "$XDG_CONFIG_HOME" +printf '{}' > "$HOME/.docker/config.json" +printf 'fixture-key\n' > "$HOME/.ssh/id_ed25519" # Create a directory that mimics a sensitive path for testing. # We use ~/.docker which is in deny_credentials. @@ -184,10 +190,7 @@ EOF cat > "$USER_PROFILES_DIR/nono-test-docker-child.json" < "$CLIENT_HOME/.claude.json" @@ -141,13 +144,28 @@ expect_success "install Claude Code npm package" \ echo "" echo "--- Claude Code Startup ---" +CLAUDE_VERSION=$(client_env claude --version) +CLAUDE_VERSION=$(capture_last_nonempty_line "$CLAUDE_VERSION") + +version_match_test "plain claude reports pinned version" "$CLAUDE_VERSION" \ + client_env claude --version + # The `claude` profile now ships as a registry pack # (always-further/claude). The client-startup smoke test exercises # the integration when both Claude Code AND the pack are installed. -# In CI environments without the pack pulled, skip the pack-dependent -# subtests cleanly rather than hit the migration prompt (which can't -# be answered non-interactively and would fail the suite). -PACK_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/nono/packages/always-further/claude" +# If the pack is installed in the real nono config, copy it into the isolated +# config used by client_env. Otherwise skip the pack-dependent subtests cleanly +# rather than hit the migration prompt, which can't be answered non-interactively. +PACK_DIR="$CLIENT_HOME/.config/nono/packages/always-further/claude" +REAL_PACK_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/nono/packages/always-further/claude" +LOCKFILE="$CLIENT_HOME/.config/nono/packages/lockfile.json" +REAL_LOCKFILE="${XDG_CONFIG_HOME:-$HOME/.config}/nono/packages/lockfile.json" +if [[ ! -f "$PACK_DIR/package.json" && -f "$REAL_PACK_DIR/package.json" && -f "$REAL_LOCKFILE" ]]; then + mkdir -p "$(dirname "$PACK_DIR")" + cp -R "$REAL_PACK_DIR" "$PACK_DIR" + cp "$REAL_LOCKFILE" "$LOCKFILE" +fi + if [[ ! -f "$PACK_DIR/package.json" ]]; then skip_test "nono run starts Claude Code successfully" \ "always-further/claude pack not installed (run 'nono pull always-further/claude' first)" @@ -157,11 +175,18 @@ if [[ ! -f "$PACK_DIR/package.json" ]]; then exit 0 fi -CLAUDE_VERSION=$(client_env claude --version) -CLAUDE_VERSION=$(capture_last_nonempty_line "$CLAUDE_VERSION") - -version_match_test "plain claude reports pinned version" "$CLAUDE_VERSION" \ - client_env claude --version +set +e +pack_check_output=$(client_env "$NONO_BIN" run --profile claude --allow-cwd --allow-net --dry-run -- claude --version &1) +pack_check_exit=$? +set -e +if [[ "$pack_check_exit" -ne 0 ]]; then + skip_test "nono run starts Claude Code successfully" \ + "always-further/claude pack is installed but failed verification" + skip_test "nono wrap starts Claude Code successfully" \ + "always-further/claude pack is installed but failed verification" + print_summary + exit 0 +fi version_match_test "nono run starts Claude Code successfully" "$CLAUDE_VERSION" \ client_env "$NONO_BIN" run --profile claude --allow-cwd --allow-net -- claude --version diff --git a/tests/integration/test_edge_cases.sh b/tests/integration/test_edge_cases.sh index c9f00bbd9..bd256c618 100755 --- a/tests/integration/test_edge_cases.sh +++ b/tests/integration/test_edge_cases.sh @@ -137,7 +137,7 @@ if is_macos; then expect_output_not_contains "grant non-existent file does not warn on macOS" "Skipping non-existent file" \ "$NONO_BIN" run --read-file /nonexistent/file.txt -- echo "should run" else - expect_output_contains "grant non-existent file is skipped with warning" "Skipping non-existent file" \ + expect_output_contains "grant non-existent file is skipped with warning" "some requested sandbox grants were skipped because the path does not exist" \ "$NONO_BIN" run --read-file /nonexistent/file.txt -- echo "should run" fi diff --git a/tests/integration/test_network.sh b/tests/integration/test_network.sh index dcddf8019..77b44a90b 100755 --- a/tests/integration/test_network.sh +++ b/tests/integration/test_network.sh @@ -42,11 +42,11 @@ else skip_test "wget blocked" "wget not installed" fi -# Note: ping requires special privileges, may not work in all environments if command_exists ping; then - # Use timeout to avoid hanging - expect_failure "ping blocked with --block-net" \ - timeout 5 "$NONO_BIN" run --block-net --allow "$TMPDIR" -- ping -c 1 -W 2 8.8.8.8 2>/dev/null || true + # Landlock ABI v4 filters TCP connect/listen, not raw ICMP sockets. On some + # Linux CI hosts ping is permitted even when TCP/UDP egress is blocked, so + # curl/wget/nc remain the meaningful --block-net assertions. + skip_test "ping blocked with --block-net" "ICMP is outside Landlock TCP network filtering" else skip_test "ping blocked" "ping not installed" fi diff --git a/tests/integration/test_profiles.sh b/tests/integration/test_profiles.sh index ac1ac4448..b5d5f44d6 100755 --- a/tests/integration/test_profiles.sh +++ b/tests/integration/test_profiles.sh @@ -33,6 +33,8 @@ trap 'cleanup_test_dir "$TMPDIR"' EXIT mkdir -p "$TMPDIR/workdir" echo "readable content" > "$TMPDIR/workdir/file.txt" +PROFILE_HOME="$TMPDIR/profile-home" +mkdir -p "$PROFILE_HOME/.config" "$PROFILE_HOME/.npm" echo "" echo "Test directory: $TMPDIR" @@ -60,6 +62,7 @@ expect_output_contains "dry-run output shows Capabilities section" "Capabilities # ~/.npm, ~/.nvm). Those live inside the collapsed system/group # block in the default dry-run output and only show with -v. expect_output_contains "node-dev profile lists Node runtime paths in dry-run -v" ".npm" \ + env HOME="$PROFILE_HOME" XDG_CONFIG_HOME="$PROFILE_HOME/.config" \ "$NONO_BIN" run -v --profile node-dev --dry-run -- echo "test" # ============================================================================= diff --git a/tests/integration/test_silent_output.sh b/tests/integration/test_silent_output.sh index b8cc094bf..82d4fb213 100755 --- a/tests/integration/test_silent_output.sh +++ b/tests/integration/test_silent_output.sh @@ -43,19 +43,17 @@ expect_output_empty() { return 1 } -# node-dev is an embedded profile that lists `$HOME/Library/pnpm` — -# a macOS-only path that's reliably absent on Linux CI runners and -# gives us a stable "missing-path" warning to assert against under -# `-v`. The previous version used `claude-code` and asserted on a -# macOS Keychain path; that profile now ships as a registry pack so -# the assertion no longer applies in this suite. -expect_output_empty \ +# node-dev is an embedded profile with optional runtime paths. Missing optional +# paths should stay hidden by default, while verbose dry-run should still expose +# capability provenance. +expect_output_not_contains \ "node-dev dry-run hides missing profile warnings by default" \ + "Profile path" \ "$NONO_BIN" run --profile node-dev --allow-cwd --dry-run -- echo ok expect_output_contains \ - "node-dev dry-run shows missing profile warnings with -v" \ - "Profile path '\$HOME/Library/pnpm' does not exist, skipping" \ + "node-dev dry-run shows verbose capability provenance with -v" \ + "[group:system_read_linux_core]" \ "$NONO_BIN" run -v --profile node-dev --allow-cwd --dry-run -- echo ok expect_output_empty \ diff --git a/tests/integration/test_system_paths.sh b/tests/integration/test_system_paths.sh index 4f0555583..d425c4925 100755 --- a/tests/integration/test_system_paths.sh +++ b/tests/integration/test_system_paths.sh @@ -137,7 +137,7 @@ if is_linux; then fi if [[ -d /sys ]]; then - expect_success "can read /sys" \ + expect_failure "broad /sys read is not granted by default" \ "$NONO_BIN" run --allow "$TMPDIR" -- ls /sys/ >/dev/null fi else diff --git a/tests/run_integration_tests.sh b/tests/run_integration_tests.sh index 2d9c80f48..7eaa484e1 100755 --- a/tests/run_integration_tests.sh +++ b/tests/run_integration_tests.sh @@ -33,8 +33,12 @@ if ! cargo build --release -p nono-cli --features test-trust-overrides 2>&1; the exit 1 fi -export NONO_BIN="$PROJECT_ROOT/target/release/nono" -export PATH="$PROJECT_ROOT/target/release:$PATH" +TARGET_DIR="${CARGO_TARGET_DIR:-$PROJECT_ROOT/target}" +[[ "$TARGET_DIR" != /* ]] && TARGET_DIR="$PROJECT_ROOT/$TARGET_DIR" +RELEASE_DIR="$TARGET_DIR/release" + +export NONO_BIN="$RELEASE_DIR/nono" +export PATH="$RELEASE_DIR:$PATH" # Verify binary exists if [[ ! -x "$NONO_BIN" ]]; then