[PLUTO-1411] Add dart/lizard/eslint/trivy/semgrep/pmd tests

.github/workflows/go.yml

@@ -49,51 +49,9 @@ jobs:
         run: |
           bash <(curl -Ls https://coverage.codacy.com/get.sh) report --force-coverage-parser go -r unit.coverage.out
 
-  ittest:
-    needs: build
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Download CLI binaries
-        uses: actions/download-artifact@v4
-        with:
-          name: cli-binaries
-          path: .
-      - name: Select correct binary
-        shell: bash
-        run: |
-          if [ "${{ matrix.os }}" = "windows-latest" ]; then
-            # Keep the .exe extension for Windows
-            echo "Using Windows binary with .exe extension"
-          elif [ "${{ matrix.os }}" = "macos-latest" ]; then
-            mv cli-v2-macos cli-v2
-          else
-            mv cli-v2-linux cli-v2
-          fi
-      - name: Make binary executable
-        if: matrix.os != 'windows-latest'
-        run: chmod +x cli-v2
-      - name: Install dependencies from .codacy/codacy.yaml
-        if: matrix.os != 'windows-latest'
-        run: |
-          ./cli-v2 install
-      - name: Install dependencies from .codacy/codacy.yaml (Windows)
-        if: matrix.os == 'windows-latest' 
-        shell: pwsh
-        run: |
-          Get-ChildItem
-          Write-Host "Current directory contents:"
-          dir
-          Write-Host "Attempting to run CLI..."
-          .\cli-v2.exe install
-
 # For now we are not releasing the CLI, as we are making some quicker iterations
   release:
-    needs: [test, ittest]
+    needs: [test]
     if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.event_name == 'push'
     runs-on: ubuntu-latest
     steps:

.github/workflows/it-test.yml

@@ -8,7 +8,11 @@
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest] # [windows-latest] removed for now
+      fail-fast: false  
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -19,40 +23,67 @@
           go-version: '1.21'
           cache: true
 
-      - name: Build CLI
+      - name: Download CLI binaries from go workflow
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: go.yml
+          name: cli-binaries
+          path: .
+
+      - name: Select correct binary
+        shell: bash
+        run: |
+          if [ "${{ matrix.os }}" = "windows-latest" ]; then
+            # Keep the .exe extension for Windows
+            echo "Using Windows binary with .exe extension"
+          elif [ "${{ matrix.os }}" = "macos-latest" ]; then
+            mv cli-v2-macos cli-v2
+          else
+            mv cli-v2-linux cli-v2
+          fi
+
+      - name: Make binary executable
+        if: matrix.os != 'windows-latest'
+        run: chmod +x cli-v2
+
+
+      - name: Run tool tests
+        if: matrix.os != 'windows-latest'
+        id: run_tests
+        continue-on-error: true
+        shell: bash
         run: |
-          go build -o cli-v2 ./cli-v2.go
-          chmod +x cli-v2
+          # Make the script executable
+          chmod +x run-tool-tests.sh
+
+          # Initialize failed tools file
+          rm -f /tmp/failed_tools.txt
+          touch /tmp/failed_tools.txt
+
+          # Run tests for each tool directory
+          for tool_dir in plugins/tools/*/; do
+            tool_name=$(basename "$tool_dir")
+            if [ -d "$tool_dir/test/src" ]; then
+              echo "Running tests for $tool_name..."
+              ./run-tool-tests.sh "$tool_name" || {
+                echo "❌ Test failed for $tool_name"
+                echo "$tool_name" >> /tmp/failed_tools.txt
+              }
+            fi
+          done
+
+          # Check if any tools failed
+          if [ -s /tmp/failed_tools.txt ] && [ "$(wc -l < /tmp/failed_tools.txt)" -gt 0 ]; then
+            echo -e "\n❌ The following tools failed their tests:"
+            cat /tmp/failed_tools.txt
+            echo "::error::Some tool tests failed. Please check the logs above for details."
+            exit 1
+          else
+            echo "✅ All tool tests passed successfully!"
+          fi
 
-      - name: Run plugin tests
+      - name: Check test results
+        if: steps.run_tests.outcome == 'failure'
         run: |
-          run_test() {
-            local tool=$1
-            local jq_filter=$2
-            echo "Running $tool tests..."
-            # Store the path to the CLI
-            CLI_PATH="$(pwd)/cli-v2"
-            # Change to test directory
-            cd plugins/tools/$tool/test/src
-            # Install the plugin
-            "$CLI_PATH" install
-            # Run analysis
-            "$CLI_PATH" analyze --tool $tool --format sarif --output actual.sarif
-            # Convert absolute paths to relative paths in the output
-            sed -i 's|file:///home/runner/work/codacy-cli-v2/codacy-cli-v2/|file:///|g' actual.sarif
-            # Compare with expected output
-            jq --sort-keys "$jq_filter" expected.sarif > expected.sorted.json
-            jq --sort-keys "$jq_filter" actual.sarif > actual.sorted.json
-            diff expected.sorted.json actual.sorted.json
-            # Go back to root directory
-            cd ../../../../..
-          }
-
-          # Run Pylint tests with simple sorting
-          run_test "pylint" "."
-
-          # Run Enigma tests with simple sorting
-          run_test "codacy-enigma-cli" "."
-
-          # Run Semgrep tests with rules sorting
-          run_test "semgrep" ".runs[0].tool.driver.rules |= if . then sort_by(.id) else . end"
+          echo "Job failed because some tool tests failed. Please check the logs above for details."
+          exit 1

plugins/tools/dartanalyzer/test/expected.sarif

@@ -0,0 +1,51 @@
+{
+  "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json",
+  "runs": [
+    {
+      "results": [
+        {
+          "locations": [
+            {
+              "physicalLocation": {
+                "artifactLocation": {
+                  "uri": "/plugins/tools/dartanalyzer/test/src/Test.dart"
+                },
+                "region": {
+                  "startLine": 2
+                }
+              }
+            }
+          ],
+          "message": {
+            "text": "Unused import: 'dart:math'."
+          },
+          "ruleId": "UNUSED_IMPORT"
+        },
+        {
+          "locations": [
+            {
+              "physicalLocation": {
+                "artifactLocation": {
+                  "uri": "/plugins/tools/dartanalyzer/test/src/Test.dart"
+                },
+                "region": {
+                  "startLine": 28
+                }
+              }
+            }
+          ],
+          "message": {
+            "text": "'oldFunction' is deprecated and shouldn't be used."
+          },
+          "ruleId": "DEPRECATED_MEMBER_USE_FROM_SAME_PACKAGE"
+        }
+      ],
+      "tool": {
+        "driver": {
+          "name": "dartanalyzer"
+        }
+      }
+    }
+  ],
+  "version": "2.1.0"
+}

plugins/tools/dartanalyzer/test/src/.codacy/codacy.yaml

@@ -0,0 +1,4 @@
+runtimes:
+    - [email protected]
+tools:
+    - [email protected]

plugins/tools/dartanalyzer/test/src/Test.dart

@@ -0,0 +1,29 @@
+// Unused import
+import 'dart:math';
+
+// Unused variable
+var unusedVar = 42;
+
+// Function with missing return type and parameter type
+foo(bar) {
+  print(bar);
+}
+
+// Function with always true condition
+void alwaysTrue() {
+  if (1 == 1) {
+    print('This is always true');
+  }
+}
+
+// Function with a deprecated member usage
+@deprecated
+void oldFunction() {
+  print('This function is deprecated');
+}
+
+void main() {
+  foo('test');
+  alwaysTrue();
+  oldFunction();
+} 

plugins/tools/eslint/test/expected.sarif

@@ -0,0 +1,54 @@
+{
+  "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json",
+  "runs": [
+    {
+      "artifacts": [
+        {
+          "location": {
+            "uri": "file:///plugins/tools/eslint/test/src/Test.js"
+          }
+        }
+      ],
+      "invocations": [
+        {
+          "executionSuccessful": false,
+          "toolConfigurationNotifications": [
+            {
+              "descriptor": {
+                "id": "ESL0999"
+              },
+              "level": "error",
+              "locations": [
+                {
+                  "physicalLocation": {
+                    "artifactLocation": {
+                      "index": 0,
+                      "uri": "file:///plugins/tools/eslint/test/src/Test.js"
+                    },
+                    "region": {
+                      "startColumn": 1,
+                      "startLine": 49
+                    }
+                  }
+                }
+              ],
+              "message": {
+                "text": "Parsing error: 'with' in strict mode"
+              }
+            }
+          ]
+        }
+      ],
+      "results": [],
+      "tool": {
+        "driver": {
+          "informationUri": "https://eslint.org",
+          "name": "ESLint",
+          "rules": [],
+          "version": "8.57.0"
+        }
+      }
+    }
+  ],
+  "version": "2.1.0"
+}

plugins/tools/eslint/test/src/.codacy/codacy.yaml

@@ -0,0 +1,5 @@
+runtimes:
+    - [email protected]
+tools:
+    - [email protected]
+

plugins/tools/eslint/test/src/.eslintrc.json

@@ -0,0 +1,16 @@
+{
+  "env": {
+    "browser": true,
+    "es2021": true,
+    "node": true
+  },
+  "extends": "eslint:recommended",
+  "parserOptions": {
+    "ecmaVersion": "latest",
+    "sourceType": "module"
+  },
+  "rules": {
+    "no-unused-vars": "warn",
+    "no-console": "warn"
+  }
+}