Skip to content

cicd: Add sanity test script #2212

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yzh119 merged 5 commits into from
Dec 16, 2025
Merged

cicd: Add sanity test script #2212

Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1b456fc
sanity test script
kahyunnam Dec 12, 2025
5a2db82
fix caat arg limit potential error
kahyunnam Dec 12, 2025
f212c33
Remove task_test_blackwell_kernels_lite.sh (merged into main script)
kahyunnam Dec 12, 2025
a749f5f
add random offset
kahyunnam Dec 15, 2025
2c2ab68
spacing nit
kahyunnam Dec 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
257 changes: 219 additions & 38 deletions scripts/task_test_blackwell_kernels.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ set -eo pipefail
: ${JUNIT_DIR:=$(realpath ./junit)}
: ${MAX_JOBS:=$(nproc)}
: ${CUDA_VISIBLE_DEVICES:=0}
: ${SAMPLE_RATE:=5} # Run every Nth test in sanity mode (5 = ~20% coverage)

# Clean Python bytecode cache to avoid stale imports (e.g., after module refactoring)
echo "Cleaning Python bytecode cache..."
Expand All @@ -16,14 +17,33 @@ echo ""
# Pytest configuration flags
PYTEST_FLAGS="--continue-on-collection-errors -s"

# Check for dry-run mode
# Parse command line arguments
DRY_RUN=false
if [[ "$1" == "--dry-run" ]] || [[ "${DRY_RUN}" == "true" ]]; then
DRY_RUN=true
SANITY_TEST=false
for arg in "$@"; do
case $arg in
--dry-run)
DRY_RUN=true
;;
--sanity-test)
SANITY_TEST=true
;;
esac
done

if [ "$DRY_RUN" = "true" ]; then
echo "🔍 DRY RUN MODE - No tests will be executed"
echo ""
fi

if [ "$SANITY_TEST" = "true" ]; then
echo "🔬 SANITY TEST MODE - Running every ${SAMPLE_RATE}th test (~$((100 / SAMPLE_RATE))% coverage)"
echo ""
else
echo "📋 FULL TEST MODE - Running all tests from each test file"
echo ""
fi

if [ "$DRY_RUN" != "true" ]; then
echo "Using CUDA version: ${CUDA_VERSION}"
echo ""
Expand Down Expand Up @@ -154,60 +174,221 @@ echo ""
FAILED_TESTS=""
TOTAL_TESTS=0
PASSED_TESTS=0
TOTAL_TEST_CASES=0
SAMPLED_TEST_CASES=0

if [ "$DRY_RUN" == "true" ]; then
echo "=========================================="
echo "DRY RUN: Tests that would be executed"
echo "=========================================="

for test_file in $TEST_FILES; do
TOTAL_TESTS=$((TOTAL_TESTS + 1))
JUNIT_FLAG="--junitxml=${JUNIT_DIR}/${test_file}.xml"
echo "$TOTAL_TESTS. pytest $PYTEST_FLAGS ${JUNIT_FLAG} \"${test_file}\""
done
if [ "$SANITY_TEST" == "true" ]; then
# Sanity test mode - show sampling details
FILE_COUNT=0
for test_file in $TEST_FILES; do
FILE_COUNT=$((FILE_COUNT + 1))

echo ""
echo "[$FILE_COUNT] Collecting tests from: $test_file"

# Temporarily disable exit on error for collection
set +e
COLLECTION_OUTPUT=$(pytest --collect-only -q "$test_file" 2>&1)
COLLECTION_EXIT_CODE=$?
set -e

ALL_NODE_IDS=$(echo "$COLLECTION_OUTPUT" | grep "::" || true)

if [ -z "$ALL_NODE_IDS" ]; then
if [ $COLLECTION_EXIT_CODE -ne 0 ]; then
echo " ⚠️ Collection failed for $test_file (skipping)"
else
echo " ⚠️ No tests found in $test_file"
fi
continue
fi

# Count total tests
TOTAL_IN_FILE=$(echo "$ALL_NODE_IDS" | wc -l)
TOTAL_TEST_CASES=$((TOTAL_TEST_CASES + TOTAL_IN_FILE))

# Sample every Nth test
SAMPLED_NODE_IDS=$(echo "$ALL_NODE_IDS" | awk "NR % $SAMPLE_RATE == 1")
SAMPLED_IN_FILE=$(echo "$SAMPLED_NODE_IDS" | wc -l)
SAMPLED_TEST_CASES=$((SAMPLED_TEST_CASES + SAMPLED_IN_FILE))

echo " Total test cases: $TOTAL_IN_FILE"
echo " Sampled test cases: $SAMPLED_IN_FILE (every ${SAMPLE_RATE}th test)"
echo " Sample of tests that would run:"
echo "$SAMPLED_NODE_IDS" | head -5 | sed 's/^/ /' || true
if [ "$SAMPLED_IN_FILE" -gt 5 ]; then
echo " ... and $((SAMPLED_IN_FILE - 5)) more"
fi
done

echo ""
echo "=========================================="
echo "DRY RUN SUMMARY (SANITY MODE)"
echo "=========================================="
echo "Total test files: $FILE_COUNT"
echo "Total test cases (full suite): $TOTAL_TEST_CASES"
echo "Sampled test cases (sanity): $SAMPLED_TEST_CASES"
if [ "$TOTAL_TEST_CASES" -gt 0 ]; then
echo "Coverage: ~$((SAMPLED_TEST_CASES * 100 / TOTAL_TEST_CASES))%"
else
echo "Coverage: N/A (no tests collected)"
fi
echo "Sample rate: every ${SAMPLE_RATE}th test"
else
# Full test mode
for test_file in $TEST_FILES; do
TOTAL_TESTS=$((TOTAL_TESTS + 1))
JUNIT_FILENAME="${test_file//\//_}.xml"
JUNIT_FLAG="--junitxml=${JUNIT_DIR}/${JUNIT_FILENAME}"
echo "$TOTAL_TESTS. pytest $PYTEST_FLAGS ${JUNIT_FLAG} \"${test_file}\""
done

echo ""
echo "=========================================="
echo "DRY RUN SUMMARY"
echo "=========================================="
echo "Total test files that would be executed: $TOTAL_TESTS"
fi

echo ""
echo "=========================================="
echo "DRY RUN SUMMARY"
echo "=========================================="
echo "Total test files that would be executed: $TOTAL_TESTS"
echo ""
echo "To actually run the tests, execute without --dry-run:"
echo " $0"
echo " $0 $([ "$SANITY_TEST" == "true" ] && echo "--sanity-test")"
echo "Or set DRY_RUN=false $0"
else
mkdir -p "${JUNIT_DIR}"
for test_file in $TEST_FILES; do
echo "=========================================="
JUNIT_FLAG="--junitxml=${JUNIT_DIR}/${test_file}.xml"
echo "Running: pytest $PYTEST_FLAGS ${JUNIT_FLAG} \"${test_file}\""
echo "=========================================="

TOTAL_TESTS=$((TOTAL_TESTS + 1))
if [ "$SANITY_TEST" == "true" ]; then
# Sanity test mode - sample tests from each file
FILE_COUNT=0

for test_file in $TEST_FILES; do
FILE_COUNT=$((FILE_COUNT + 1))

echo "=========================================="
echo "[$FILE_COUNT] Processing: $test_file"
echo "=========================================="

# Collect all test node IDs for this file
echo "Collecting test cases..."

if pytest $PYTEST_FLAGS "${JUNIT_FLAG}" "${test_file}"; then
echo "✅ PASSED: $test_file"
PASSED_TESTS=$((PASSED_TESTS + 1))
# Temporarily disable exit on error for collection
set +e
COLLECTION_OUTPUT=$(pytest --collect-only -q "$test_file" 2>&1)
COLLECTION_EXIT_CODE=$?
set -e

ALL_NODE_IDS=$(echo "$COLLECTION_OUTPUT" | grep "::" || true)

if [ -z "$ALL_NODE_IDS" ]; then
if [ $COLLECTION_EXIT_CODE -ne 0 ]; then
echo "⚠️ Collection failed for $test_file (skipping)"
else
echo "⚠️ No tests found in $test_file"
fi
echo ""
continue
fi

# Count total tests
TOTAL_IN_FILE=$(echo "$ALL_NODE_IDS" | wc -l)
TOTAL_TEST_CASES=$((TOTAL_TEST_CASES + TOTAL_IN_FILE))

# Sample every Nth test
SAMPLED_NODE_IDS=$(echo "$ALL_NODE_IDS" | awk "NR % $SAMPLE_RATE == 1")
SAMPLED_IN_FILE=$(echo "$SAMPLED_NODE_IDS" | wc -l)
SAMPLED_TEST_CASES=$((SAMPLED_TEST_CASES + SAMPLED_IN_FILE))

echo "Total test cases in file: $TOTAL_IN_FILE"
echo "Running sampled test cases: $SAMPLED_IN_FILE (every ${SAMPLE_RATE}th test)"

if [ "$SAMPLED_IN_FILE" -eq 0 ]; then
echo "⚠️ No tests sampled from $test_file, skipping"
echo ""
continue
fi

# Create a bash array with the node IDs
mapfile -t SAMPLED_NODE_IDS_ARRAY <<< "$SAMPLED_NODE_IDS"

JUNIT_FILENAME="${test_file//\//_}.xml"
JUNIT_FLAG="--junitxml=${JUNIT_DIR}/${JUNIT_FILENAME}"

# Run pytest with the sampled node IDs
TOTAL_TESTS=$((TOTAL_TESTS + 1))

if pytest $PYTEST_FLAGS "${JUNIT_FLAG}" "${SAMPLED_NODE_IDS_ARRAY[@]}"; then
echo "✅ PASSED: $test_file ($SAMPLED_IN_FILE/$TOTAL_IN_FILE tests)"
PASSED_TESTS=$((PASSED_TESTS + 1))
else
echo "❌ FAILED: $test_file ($SAMPLED_IN_FILE/$TOTAL_IN_FILE tests)"
FAILED_TESTS="$FAILED_TESTS\n - $test_file"
EXIT_CODE=1
fi

echo ""
done

echo "=========================================="
echo "SANITY TEST SUMMARY"
echo "=========================================="
echo "Total test files executed: $TOTAL_TESTS"
echo "Test files passed: $PASSED_TESTS"
echo "Test files failed: $((TOTAL_TESTS - PASSED_TESTS))"
echo ""
echo "Total test cases (full suite): $TOTAL_TEST_CASES"
echo "Sampled test cases (executed): $SAMPLED_TEST_CASES"
if [ "$TOTAL_TEST_CASES" -gt 0 ]; then
echo "Coverage: ~$((SAMPLED_TEST_CASES * 100 / TOTAL_TEST_CASES))%"
else
echo "❌ FAILED: $test_file"
FAILED_TESTS="$FAILED_TESTS\n - $test_file"
EXIT_CODE=1
echo "Coverage: N/A (no tests collected)"
fi
echo "Sample rate: every ${SAMPLE_RATE}th test"

echo ""
done
if [ -n "$FAILED_TESTS" ]; then
echo ""
echo "Failed test files:"
echo -e "$FAILED_TESTS"
fi
else
# Full test mode - run all tests in each file
for test_file in $TEST_FILES; do
echo "=========================================="
JUNIT_FILENAME="${test_file//\//_}.xml"
JUNIT_FLAG="--junitxml=${JUNIT_DIR}/${JUNIT_FILENAME}"
echo "Running: pytest $PYTEST_FLAGS ${JUNIT_FLAG} \"${test_file}\""
echo "=========================================="

TOTAL_TESTS=$((TOTAL_TESTS + 1))

if pytest $PYTEST_FLAGS "${JUNIT_FLAG}" "${test_file}"; then
echo "✅ PASSED: $test_file"
PASSED_TESTS=$((PASSED_TESTS + 1))
else
echo "❌ FAILED: $test_file"
FAILED_TESTS="$FAILED_TESTS\n - $test_file"
EXIT_CODE=1
fi

echo "=========================================="
echo "TEST SUMMARY"
echo "=========================================="
echo "Total test files executed: $TOTAL_TESTS"
echo "Passed: $PASSED_TESTS"
echo "Failed: $((TOTAL_TESTS - PASSED_TESTS))"
echo ""
done

if [ -n "$FAILED_TESTS" ]; then
echo ""
echo "Failed tests:"
echo -e "$FAILED_TESTS"
echo "=========================================="
echo "TEST SUMMARY"
echo "=========================================="
echo "Total test files executed: $TOTAL_TESTS"
echo "Passed: $PASSED_TESTS"
echo "Failed: $((TOTAL_TESTS - PASSED_TESTS))"

if [ -n "$FAILED_TESTS" ]; then
echo ""
echo "Failed tests:"
echo -e "$FAILED_TESTS"
fi
fi
fi

Expand Down