Skip to content

Commit 05bbd13

Browse files
EforenEforen
committed
Sort out some testing stuff
1 parent 51e427a commit 05bbd13

14 files changed

Lines changed: 8416 additions & 24 deletions

File tree

‎GenerateKuick-trx.sh‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
#!/bin/bash
2+
3+
# Simple script to generate TRX file for all Kuick tests
4+
# Usage: ./GenerateKuick-trx.sh
5+
# Output: TestResults/Kuick.trx
6+
7+
set -e
8+
9+
# Get the directory where this script is located (repo root)
10+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
11+
REPO_ROOT="$SCRIPT_DIR"
12+
13+
# CD into the repo root to ensure we're in the correct location
14+
cd "$REPO_ROOT"
15+
16+
echo "Working from repo root: $REPO_ROOT"
17+
18+
# Ensure output directory exists
19+
mkdir -p TestResults
20+
21+
echo "Generating Kuick test results..."
22+
23+
# Change to source directory
24+
cd src
25+
26+
# Build and run all tests, output TRX to workspace root
27+
dotnet test Kore.Kuick.Tests/Kore.Kuick.Tests.csproj --configuration Release --verbosity minimal --logger "trx;LogFileName=$(pwd)/../TestResults/Kuick.trx" | true
28+
29+
# Return to workspace root
30+
cd "$REPO_ROOT"
31+
32+
echo "TRX file generated at TestResults/Kuick.trx"
33+
34+
# Remove absolute paths from TRX file to make it generic
35+
echo "Removing absolute paths from TRX file..."
36+
if [[ -f "TestResults/Kuick.trx" ]]; then
37+
# Replace all instances of the repo root path and its subdirectories with relative paths (case-insensitive)
38+
# This handles cases like /full/path/to/repo/src/ -> src/
39+
sed -i "s|$REPO_ROOT/||gI" "TestResults/Kuick.trx"
40+
# Also handle case where repo root appears without trailing slash (case-insensitive)
41+
sed -i "s|$REPO_ROOT||gI" "TestResults/Kuick.trx"
42+
43+
echo "Absolute paths removed from TRX file"
44+
else
45+
echo "Warning: TRX file not found at TestResults/Kuick.trx"
46+
fi
47+
48+
# Generate markdown summary
49+
echo "Generating markdown summary..."
50+
./GenerateSummary.sh
51+
52+
# Strip GUIDs and timestamps from TRX file and convert to XML
53+
echo "Stripping GUIDs and timestamps from TRX file..."
54+
if [[ -f "TestResults/Kuick.trx" ]]; then
55+
56+
# Strip GUIDs and timestamps, convert to XML with sorted test results
57+
./strip-xml-guids.sh --sort "TestResults/Kuick.trx" "TestResults/Kuick.xml"
58+
59+
# Remove the original TRX file since we now have the clean XML version
60+
rm "TestResults/Kuick.trx"
61+
62+
echo "Clean XML file generated at TestResults/Kuick.xml"
63+
else
64+
echo "Warning: TRX file not found, skipping GUID/timestamp stripping"
65+
fi
66+
67+
echo "Done!"

‎GenerateSummary.sh‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
#!/bin/bash
2+
3+
# Script to generate a markdown summary from Kuick TRX test results
4+
# Usage: ./GenerateSummary.sh
5+
# Input: TestResults/Kuick.trx or TestResults/Kuick.trx.backup
6+
# Output: TestResults/KUICK.md
7+
8+
set -e
9+
10+
TRX_FILE="TestResults/Kuick.trx"
11+
BACKUP_TRX_FILE="TestResults/Kuick.trx.backup"
12+
XML_FILE="TestResults/Kuick.xml"
13+
OUTPUT_FILE="TestResults/KUICK.md"
14+
15+
# Check for TRX file, fall back to backup, then XML if neither TRX exists
16+
if [ -f "$TRX_FILE" ]; then
17+
SOURCE_FILE="$TRX_FILE"
18+
elif [ -f "$BACKUP_TRX_FILE" ]; then
19+
SOURCE_FILE="$BACKUP_TRX_FILE"
20+
elif [ -f "$XML_FILE" ]; then
21+
SOURCE_FILE="$XML_FILE"
22+
else
23+
echo "Error: No test results file found at $TRX_FILE, $BACKUP_TRX_FILE, or $XML_FILE"
24+
echo "Please run ./GenerateKuick-trx.sh first"
25+
exit 1
26+
fi
27+
28+
echo "Generating markdown summary from $SOURCE_FILE..."
29+
30+
# Create output directory if it doesn't exist
31+
mkdir -p TestResults
32+
33+
# Parse TRX file and generate markdown
34+
cat > "$OUTPUT_FILE" << 'EOF'
35+
# Kuick Test Results Summary
36+
37+
## Overall Statistics
38+
39+
EOF
40+
41+
# Extract overall statistics
42+
TOTAL_TESTS=$(grep -o 'total="[0-9]*"' "$SOURCE_FILE" | head -1 | sed 's/total="//;s/"//')
43+
EXECUTED_TESTS=$(grep -o 'executed="[0-9]*"' "$SOURCE_FILE" | head -1 | sed 's/executed="//;s/"//')
44+
PASSED_TESTS=$(grep -o 'passed="[0-9]*"' "$SOURCE_FILE" | head -1 | sed 's/passed="//;s/"//')
45+
FAILED_TESTS=$(grep -o 'failed="[0-9]*"' "$SOURCE_FILE" | head -1 | sed 's/failed="//;s/"//')
46+
47+
echo "| Metric | Count | Status |" >> "$OUTPUT_FILE"
48+
echo "|--------|-------|--------|" >> "$OUTPUT_FILE"
49+
echo "| Total Tests | $TOTAL_TESTS | ℹ️ |" >> "$OUTPUT_FILE"
50+
echo "| Executed | $EXECUTED_TESTS | ▶️ |" >> "$OUTPUT_FILE"
51+
echo "| Passed | $PASSED_TESTS | ✅ |" >> "$OUTPUT_FILE"
52+
echo "| Failed | $FAILED_TESTS | ❌ |" >> "$OUTPUT_FILE"
53+
echo "" >> "$OUTPUT_FILE"
54+
55+
# Add test class breakdown
56+
echo "## Test Method Breakdown" >> "$OUTPUT_FILE"
57+
echo "" >> "$OUTPUT_FILE"
58+
echo "| Status | Details | Test Method |" >> "$OUTPUT_FILE"
59+
echo "|--------|---------|-------------|" >> "$OUTPUT_FILE"
60+
61+
# Create temporary files for processing
62+
TEMP_CLASSES=$(mktemp)
63+
TEMP_SORTED=$(mktemp)
64+
65+
# Extract test results and group by class
66+
grep -o '<UnitTestResult.*testName="[^"]*".*outcome="[^"]*"' "$SOURCE_FILE" | while IFS= read -r line; do
67+
TEST_NAME=$(echo "$line" | sed 's/.*testName="//;s/".*$//' | sed 's/&quot;/"/g;s/&amp;/\&/g;s/&lt;/</g;s/&gt;/>/g')
68+
OUTCOME=$(echo "$line" | sed 's/.*outcome="//;s/".*$//')
69+
70+
# Extract method name (everything before the first parenthesis)
71+
METHOD_NAME=$(echo "$TEST_NAME" | sed 's/(.*//')
72+
73+
echo "$METHOD_NAME|$OUTCOME"
74+
done > "$TEMP_CLASSES"
75+
76+
# Sort by class name and count occurrences
77+
sort "$TEMP_CLASSES" | uniq -c | sort -k3 > "$TEMP_SORTED"
78+
79+
# Process sorted results
80+
while read count class_outcome; do
81+
METHOD=$(echo "$class_outcome" | cut -d'|' -f1)
82+
OUTCOME=$(echo "$class_outcome" | cut -d'|' -f2)
83+
84+
if [ "$OUTCOME" = "Passed" ]; then
85+
EMOJI="✅"
86+
STATUS="$count passed"
87+
else
88+
EMOJI="❌"
89+
STATUS="$count failed"
90+
fi
91+
92+
echo "| $EMOJI | $STATUS | \`$METHOD\` |" >> "$OUTPUT_FILE"
93+
done < "$TEMP_SORTED"
94+
95+
# Clean up temporary files
96+
rm -f "$TEMP_CLASSES" "$TEMP_SORTED"
97+
98+
# Add detailed test results
99+
echo "## Detailed Test Results" >> "$OUTPUT_FILE"
100+
echo "" >> "$OUTPUT_FILE"
101+
echo "| Status | Error Details | Test Name |" >> "$OUTPUT_FILE"
102+
echo "|--------|---------------|-----------|" >> "$OUTPUT_FILE"
103+
104+
# Create temporary file for test results
105+
TEMP_TESTS=$(mktemp)
106+
107+
# Extract all test results with details
108+
grep -o '<UnitTestResult.*testName="[^"]*".*outcome="[^"]*"' "$SOURCE_FILE" | while IFS= read -r line; do
109+
TEST_NAME=$(echo "$line" | sed 's/.*testName="//;s/".*$//' | sed 's/&quot;/"/g;s/&amp;/\&/g;s/&lt;/</g;s/&gt;/>/g')
110+
OUTCOME=$(echo "$line" | sed 's/.*outcome="//;s/".*$//')
111+
112+
# Clean up test name (remove class prefix and parameters)
113+
CLEAN_NAME=$(echo "$TEST_NAME" | sed 's/.*\.\([^.]*\)(\(.*\))/\1(\2)/')
114+
115+
if [ "$OUTCOME" = "Passed" ]; then
116+
EMOJI="✅"
117+
ERROR_DETAILS="N/A"
118+
else
119+
EMOJI="❌"
120+
ERROR_DETAILS="Test failed - see full TRX for details"
121+
fi
122+
123+
echo "$CLEAN_NAME|$EMOJI|$ERROR_DETAILS"
124+
done > "$TEMP_TESTS"
125+
126+
# Sort by test name and output to markdown
127+
sort "$TEMP_TESTS" | while IFS='|' read -r clean_name emoji error_details; do
128+
echo "| $emoji | $error_details | \`$clean_name\` |" >> "$OUTPUT_FILE"
129+
done
130+
131+
# Clean up temporary file
132+
rm -f "$TEMP_TESTS"
133+
134+
echo "" >> "$OUTPUT_FILE"
135+
echo "---" >> "$OUTPUT_FILE"
136+
echo "*Generated on $(date)*" >> "$OUTPUT_FILE"
137+
echo "" >> "$OUTPUT_FILE"
138+
echo "📁 **Full results**: [Kuick.xml](./Kuick.xml)" >> "$OUTPUT_FILE"
139+
140+
echo "✅ Markdown summary generated at $OUTPUT_FILE"

‎README-test-tools.md‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
# Kuick Test Tools
2+
3+
This directory contains scripts for running and analyzing Kuick parser and lexer tests.
4+
5+
## Scripts Overview
6+
7+
### 🔧 `GenerateKuick-trx.sh`
8+
Runs all Kuick tests and generates TRX results with markdown summary.
9+
10+
**Usage:**
11+
```bash
12+
./GenerateKuick-trx.sh
13+
```
14+
15+
**Output:**
16+
- `TestResults/Kuick.trx` - Full test results in TRX format
17+
- `TestResults/KUICK.md` - Markdown summary with pass/fail statistics
18+
19+
### 🔍 `SearchKuick-trx.sh`
20+
Search for specific test patterns in the TRX results.
21+
22+
**Usage:**
23+
```bash
24+
./SearchKuick-trx.sh "search_term" [--summary]
25+
```
26+
27+
**Examples:**
28+
```bash
29+
# Search for CSR-related tests
30+
./SearchKuick-trx.sh "csrr"
31+
32+
# Search for rdcycle tests with full summary
33+
./SearchKuick-trx.sh "rdcycle" --summary
34+
35+
# Search for floating-point tests
36+
./SearchKuick-trx.sh "f[0-9]"
37+
```
38+
39+
### 📊 `GenerateSummary.sh`
40+
Generate markdown summary from existing TRX file.
41+
42+
**Usage:**
43+
```bash
44+
./GenerateSummary.sh
45+
```
46+
47+
**Output:**
48+
- `TestResults/KUICK.md` - Markdown summary with emoji status indicators
49+
50+
## Test Results Format
51+
52+
The markdown summary includes:
53+
54+
### Overall Statistics
55+
| Metric | Count | Status |
56+
|--------|-------|--------|
57+
| Total Tests | 879 | ℹ️ |
58+
| Executed | 855 | ▶️ |
59+
| Passed | 772 | ✅ |
60+
| Failed | 83 | ❌ |
61+
62+
### Status Indicators
63+
- ✅ **Passed** - Test completed successfully
64+
- ❌ **Failed** - Test failed with errors
65+
- ℹ️ **Info** - General information
66+
- ▶️ **Executed** - Tests that were run
67+
68+
## Common Test Patterns
69+
70+
### CSR (Control Status Register) Tests
71+
```bash
72+
# All CSR-related tests
73+
./SearchKuick-trx.sh "csr"
74+
75+
# Specific CSR instructions
76+
./SearchKuick-trx.sh "csrr"
77+
./SearchKuick-trx.sh "csrw"
78+
./SearchKuick-trx.sh "rdcycle"
79+
```
80+
81+
### Pseudo Instructions
82+
```bash
83+
# All pseudo instructions
84+
./SearchKuick-trx.sh "PseudoInstructions"
85+
86+
# Specific pseudo instructions
87+
./SearchKuick-trx.sh "nop"
88+
./SearchKuick-trx.sh "neg"
89+
./SearchKuick-trx.sh "beqz"
90+
```
91+
92+
### Floating Point Tests
93+
```bash
94+
# All floating point registers
95+
./SearchKuick-trx.sh "f[0-9]"
96+
97+
# Floating point operations
98+
./SearchKuick-trx.sh "flw"
99+
./SearchKuick-trx.sh "fld"
100+
./SearchKuick-trx.sh "fsw"
101+
```
102+
103+
## Workflow
104+
105+
1. **Run tests:** `./GenerateKuick-trx.sh`
106+
2. **Check specific failures:** `./SearchKuick-trx.sh "failing_pattern"`
107+
3. **View full summary:** Open `TestResults/KUICK.md`
108+
109+
## Files Generated
110+
111+
- `TestResults/Kuick.trx` - Full XML test results
112+
- `TestResults/KUICK.md` - Human-readable markdown summary
113+
114+
## Dependencies
115+
116+
- .NET SDK (for running tests)
117+
- `grep`, `sed` (for text processing)
118+
- `bash` (shell environment)
119+
120+
---
121+
*Generated for the Kuick RISC-V Assembler project*

0 commit comments

Comments
 (0)