diff --git a/.github/workflows/comment-revalidation.yml b/.github/workflows/comment-revalidation.yml index 69869f5..f725c3c 100644 --- a/.github/workflows/comment-revalidation.yml +++ b/.github/workflows/comment-revalidation.yml @@ -25,11 +25,16 @@ on: required: false type: string default: '' - gemini-model: - description: 'Gemini model to use' + provider: + description: 'LLM provider (openrouter, anthropic, openai, auto)' required: false type: string - default: 'gemini-2.5-flash' + default: 'auto' + model: + description: 'Model to use' + required: false + type: string + default: 'anthropic/claude-haiku-4.5' comment-title: description: 'Title for the validation comment' required: false @@ -41,9 +46,15 @@ on: type: boolean default: false secrets: - GEMINI_SERVICE_ACCOUNT_KEY: - description: 'Base64-encoded Gemini service account key' - required: true + OPENROUTER_API_KEY: + description: 'OpenRouter API key (or ANTHROPIC_API_KEY/OPENAI_API_KEY)' + required: false + ANTHROPIC_API_KEY: + description: 'Anthropic API key' + required: false + OPENAI_API_KEY: + description: 'OpenAI API key' + required: false USABLE_API_TOKEN: description: 'Usable API token' required: true @@ -273,13 +284,16 @@ jobs: prompt-fragment-id: ${{ inputs.prompt-fragment-id || '' }} workspace-id: ${{ inputs.workspace-id || '60c10ca2-4115-4c1a-b6d7-04ac39fd3938' }} base-ref: ${{ steps.base-ref.outputs.base-ref }} - gemini-model: ${{ inputs.gemini-model || 'gemini-2.5-flash' }} + provider: ${{ inputs.provider || 'auto' }} + model: ${{ inputs.model || 'anthropic/claude-haiku-4.5' }} comment-title: ${{ inputs.comment-title || '๐Ÿ”„ Revalidation (Comment Triggered)' }} comment-mode: 'create' fail-on-critical: ${{ inputs.fail-on-critical || false }} override-comment: ${{ needs.check-trigger.outputs.comment-body }} env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} COMMENT_AUTHOR: ${{ github.event.comment.user.login }} diff --git a/.github/workflows/test-reusable-workflow.yml b/.github/workflows/test-reusable-workflow.yml index 3198d74..d7c8715 100644 --- a/.github/workflows/test-reusable-workflow.yml +++ b/.github/workflows/test-reusable-workflow.yml @@ -23,11 +23,12 @@ jobs: with: workspace-id: '60c10ca2-4115-4c1a-b6d7-04ac39fd3938' prompt-file: 'templates/basic-validation.md' - gemini-model: 'gemini-2.5-flash' + provider: 'auto' + model: 'anthropic/claude-haiku-4.5' comment-title: '๐Ÿงช Reusable Workflow Test' fail-on-critical: false secrets: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} permissions: contents: read @@ -62,22 +63,26 @@ jobs: **Test Configuration**: - Workspace ID: 60c10ca2-4115-4c1a-b6d7-04ac39fd3938 - Prompt File: templates/basic-validation.md - - Model: gemini-2.5-flash + - Provider: auto + - Model: anthropic/claude-haiku-4.5 - Comment Title: ๐Ÿงช Reusable Workflow Test - + **Result**: ${{ needs.test-reusable.result }} - + **Next Steps**: - If successful: Reusable workflow is ready for users - If failed: Check workflow logs for errors - + **How Users Will Call It**: \`\`\`yaml uses: flowcore/usable-pr-validator/.github/workflows/comment-revalidation.yml@v1 with: workspace-id: 'their-workspace-uuid' + # Optional - defaults to auto and anthropic/claude-haiku-4.5 + # provider: 'openrouter' # or 'anthropic', 'openai', 'auto' + # model: 'anthropic/claude-3.7-sonnet' # for higher quality secrets: - GEMINI_SERVICE_ACCOUNT_KEY: \${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: \${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: \${{ secrets.USABLE_API_TOKEN }} \`\`\` EOF diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d821987..fd3ace3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -182,22 +182,109 @@ jobs: - name: Verify Cleanup Steps run: | # Check that action.yml has cleanup steps - if ! grep -q "rm -f /tmp/service-account.json" action.yml; then - echo "::error::Missing cleanup for service-account.json" + if ! grep -q "rm -f /tmp/forge-config.yaml" action.yml; then + echo "::error::Missing cleanup for forge-config.yaml" exit 1 fi - - if ! grep -q "rm -f /tmp/gemini-settings.json" action.yml; then - echo "::error::Missing cleanup for gemini-settings.json" + + if ! grep -q "rm -f /tmp/validation-.*\.md" action.yml; then + echo "::error::Missing cleanup for validation files" exit 1 fi - + echo "โœ… Cleanup steps verified" + test-mcp-connection: + name: Test MCP Connection + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install ForgeCode + run: | + echo "Installing ForgeCode CLI..." + npm install -g forgecode@latest + forge --version + + echo "" + echo "Node.js version:" + node --version + echo "npm version:" + npm --version + + - name: Setup MCP Server + env: + USABLE_URL: ${{ vars.USABLE_URL || 'https://usable.dev' }} + USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} + WORKSPACE_ID: '60c10ca2-4115-4c1a-b6d7-04ac39fd3938' + run: | + ./scripts/setup-mcp.sh + + - name: Verify MCP Server Package + run: | + echo "Testing if @usabledev/mcp-server can be executed..." + npx --yes @usabledev/mcp-server@latest --version || echo "::warning::MCP server version check failed" + + echo "" + echo "Checking .mcp.json configuration..." + echo "File exists: $([ -f .mcp.json ] && echo 'YES' || echo 'NO')" + + if [ -f .mcp.json ]; then + echo "" + echo "Full .mcp.json content (with token partially masked for security):" + cat .mcp.json | sed -E 's/(mmesh_[a-zA-Z0-9_]{6})[a-zA-Z0-9_]*/\1***/g' + + echo "" + echo "Parsed configuration:" + jq '.' .mcp.json 2>&1 || echo "::warning::Failed to parse JSON" + + echo "" + echo "Command that will be executed:" + jq -r '.mcpServers.usable.command + " " + (.mcpServers.usable.args | join(" "))' .mcp.json 2>&1 || echo "::warning::Failed to extract command" + + echo "" + echo "Environment variables configured in .mcp.json:" + jq -r '.mcpServers.usable.env | keys[]' .mcp.json 2>&1 || echo "::warning::Failed to extract env vars" + else + echo "::error::.mcp.json file not found!" + fi + + echo "" + echo "ForgeCode MCP servers:" + forge mcp list + + - name: Test MCP Connection + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} + USABLE_URL: ${{ vars.USABLE_URL || 'https://usable.dev' }} + PROVIDER: ${{ vars.PROVIDER || 'auto' }} + MODEL: ${{ vars.MODEL || 'anthropic/claude-haiku-4.5' }} + run: | + ./scripts/test-mcp-connection.sh + + - name: Upload MCP Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: mcp-test-results + path: | + /tmp/mcp-test-*.txt + /tmp/mcp-test-*.json + retention-days: 7 + if-no-files-found: warn + integration-test: name: Integration Test runs-on: ubuntu-latest if: github.event_name == 'pull_request' + needs: test-mcp-connection permissions: contents: read pull-requests: write @@ -238,6 +325,21 @@ jobs: ## Your Task Validate this PR for the usable-pr-validator GitHub Action repository. + ### Step 0: Verify MCP Connection (REQUIRED - CRITICAL) + **YOU MUST TEST THE USABLE MCP CONNECTION FIRST:** + + 1. Call the `mcp_usable_list-workspaces` tool with `{"includeArchived": false}` + 2. In your report, include the EXACT output showing: + - Total number of workspaces found + - Name of each workspace + - Whether the expected workspace "Flowcore" (ID: 60c10ca2-4115-4c1a-b6d7-04ac39fd3938) was found + + **CRITICAL**: If you do not call this tool and show the results, your report will be considered INVALID. + + Expected result: Should find at least the "Flowcore" workspace. + + If the MCP connection fails, report this as a CRITICAL VIOLATION with the exact error message. + ### Step 1: Get PR Changes ```bash # Compare base ref (branch or tag) with HEAD @@ -279,6 +381,22 @@ jobs: # PR Validation Report + ## MCP Connection Verification โœ… (REQUIRED) + + **MCP Server**: https://usable.dev/api/mcp + + ### Connection Test Results: + - **Status**: Connected โœ… or Failed โŒ + - **Tool Called**: `mcp_usable_list-workspaces` + - **Total Workspaces Found**: [exact number] + - **Workspaces**: + - [Workspace 1 name] (ID: [uuid]) + - [Workspace 2 name] (ID: [uuid]) + - ... + - **Target Workspace Found**: โœ… Flowcore (60c10ca2-4115-4c1a-b6d7-04ac39fd3938) or โŒ Not Found + + **Verification Stamp**: ๐Ÿ” MCP connection verified on [timestamp] + ## Summary [Brief overview of the changes and validation results] @@ -313,7 +431,7 @@ jobs: comment-mode: 'update' fail-on-critical: false env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} - name: Debug Test Results @@ -321,7 +439,7 @@ jobs: run: | echo "Test result: ${{ steps.test-action.outcome }}" if [ -f "/tmp/validation-full-output.md" ]; then - echo "::group::Full Gemini Output" + echo "::group::Full ForgeCode Output" cat /tmp/validation-full-output.md echo "::endgroup::" else @@ -332,6 +450,7 @@ jobs: name: Integration Test (Documentation) runs-on: ubuntu-latest if: github.event_name == 'pull_request' + needs: test-mcp-connection permissions: contents: read pull-requests: write @@ -372,6 +491,12 @@ jobs: ## Your Task Validate documentation changes in this PR. + ### Step 0: Verify MCP Connection (REQUIRED) + **YOU MUST call `mcp_usable_list-workspaces` tool first** and show the results in your report with: + - Total workspaces found + - List of workspace names and IDs + - Confirmation that "Flowcore" workspace was found + ### Get Changes ```bash # Compare base ref (branch or tag) with HEAD for markdown files @@ -399,6 +524,12 @@ jobs: # PR Validation Report + ## MCP Connection Verification โœ… + - **Status**: Connected โœ… or Failed โŒ + - **Workspaces Found**: [count] + - **Target Workspace**: โœ… Flowcore found or โŒ Not Found + - **Verification Stamp**: ๐Ÿ” MCP verified + ## Summary [Overview of documentation changes] @@ -430,7 +561,7 @@ jobs: comment-mode: 'update' fail-on-critical: false env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} - name: Debug Documentation Results @@ -438,7 +569,7 @@ jobs: run: | echo "Docs test result: ${{ steps.docs-action.outcome }}" if [ -f "/tmp/validation-full-output.md" ]; then - echo "::group::Full Gemini Output" + echo "::group::Full ForgeCode Output" cat /tmp/validation-full-output.md echo "::endgroup::" else @@ -449,6 +580,7 @@ jobs: name: Integration Test (Comment Revalidation) runs-on: ubuntu-latest if: github.event_name == 'pull_request' + needs: test-mcp-connection permissions: contents: read pull-requests: write @@ -489,6 +621,11 @@ jobs: ## Your Task Test the comment revalidation feature. + ### Step 0: Verify MCP Connection (REQUIRED) + **YOU MUST call `mcp_usable_list-workspaces` tool first** and show the results in your report with: + - Total workspaces found + - Verification that "Flowcore" workspace is accessible + ### Step 1: Verify Override Comment Check if there is an override/clarification comment in the PR context above (marked with ๐Ÿ”„). @@ -515,6 +652,11 @@ jobs: # PR Validation Report + ## MCP Connection Verification โœ… + - **Status**: Connected โœ… or Failed โŒ + - **Workspaces Found**: [count] + - **Verification Stamp**: ๐Ÿ” MCP verified + ## Summary [Mention this is a comment revalidation test] [Note if override comment was detected] @@ -569,7 +711,7 @@ jobs: comment-mode: 'update' fail-on-critical: false env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} COMMENT_AUTHOR: 'test-user' diff --git a/.gitignore b/.gitignore index 44e071d..4904c67 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .DS_Store .AppleDouble .LSOverride +.mcp.json # Secrets and credentials *.json.key diff --git a/README.md b/README.md index 074b038..9ed914f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ๐Ÿค– Usable PR Validator -> Validate Pull Requests against your Usable knowledge base standards using Google Gemini AI +> Validate Pull Requests against your Usable knowledge base standards using AI (OpenRouter, Anthropic, OpenAI, and more) [![GitHub Marketplace](https://img.shields.io/badge/Marketplace-Usable%20PR%20Validator-blue?logo=github)](https://github.com/marketplace/actions/usable-pr-validator) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) @@ -8,9 +8,10 @@ ## โœจ Features -- ๐Ÿง  **AI-Powered Validation**: Uses Google Gemini to understand context and architectural patterns +- ๐Ÿง  **AI-Powered Validation**: Uses advanced AI models (Claude, GPT-4, Llama, etc.) via ForgeCode +- ๐Ÿ”Œ **Multi-Provider Support**: OpenRouter, Anthropic, OpenAI, and more - bring your own API key - ๐Ÿ“š **Usable Integration**: Validate PRs against your team's knowledge base stored in Usable -- ๐Ÿ”Œ **MCP Protocol**: Connects directly to Usable's MCP server for real-time standards +- ๐Ÿ”— **MCP Protocol**: Connects directly to Usable's MCP server for real-time standards - ๐ŸŽฏ **System Prompts**: Organization-wide validation standards fetched from Usable and auto-merged - ๐Ÿš€ **Dynamic Prompts**: Fetch latest validation prompts from Usable API (no static files needed!) - ๐Ÿ’ฌ **Comment-Triggered Revalidation**: Mention `@usable` in PR comments to trigger revalidation with context @@ -25,10 +26,12 @@ ### Prerequisites -1. A Google Cloud project with Vertex AI API enabled -2. A service account key with Vertex AI permissions -3. A Usable account with API token ([get one at usable.dev](https://usable.dev)) -4. GitHub repository with pull requests +1. An API key from one of: + - [OpenRouter](https://openrouter.ai) (recommended - access to 200+ models) + - [Anthropic](https://console.anthropic.com) (Claude models) + - [OpenAI](https://platform.openai.com) (GPT models) +2. A Usable account with API token ([get one at usable.dev](https://usable.dev)) +3. GitHub repository with pull requests ### Step 1: Create Validation Prompt @@ -57,12 +60,7 @@ git diff origin/{{BASE_BRANCH}}...{{HEAD_BRANCH}} Go to your repository Settings โ†’ Secrets โ†’ Actions and add: -- `GEMINI_SERVICE_ACCOUNT_KEY`: Base64-encoded service account JSON key - - ```bash - cat service-account.json | base64 - ``` - +- `OPENROUTER_API_KEY`: Your OpenRouter API key (or `ANTHROPIC_API_KEY` / `OPENAI_API_KEY`) - `USABLE_API_TOKEN`: Your Usable API token (get from [usable.dev](https://usable.dev) โ†’ Settings โ†’ API Tokens) ### Step 3: Create Workflow @@ -87,18 +85,89 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - + - uses: flowcore/usable-pr-validator@latest with: prompt-file: '.github/prompts/pr-validation.md' workspace-id: 'your-workspace-uuid' + # Optional: specify provider and model (defaults to auto-detect and anthropic/claude-haiku-4.5) + # provider: 'openrouter' # or 'anthropic', 'openai', 'auto' + # model: 'anthropic/claude-3.7-sonnet' # for higher quality env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} ``` That's it! Your PRs will now be validated automatically. ๐ŸŽ‰ +## ๐Ÿงช Local Testing + +You can test the PR validator locally before pushing to GitHub Actions: + +### Required Environment Variables + +```bash +# At least one of these API keys (depending on your provider) +export OPENROUTER_API_KEY='your-openrouter-key' # Recommended +# OR +export ANTHROPIC_API_KEY='your-anthropic-key' # For direct Anthropic +# OR +export OPENAI_API_KEY='your-openai-key' # For OpenAI + +# Required +export USABLE_API_TOKEN='your-usable-token' + +# Optional (can be set in the script or use defaults) +export USABLE_URL='https://usable.dev' # Usable instance URL (default) +export PROVIDER='auto' # or 'openrouter', 'anthropic', 'openai' +export MODEL='anthropic/claude-haiku-4.5' # default model (fast and cost-effective) +export WORKSPACE_ID='your-workspace-uuid' # Usable workspace +export BASE_BRANCH='main' # base branch for diff +export HEAD_BRANCH='feature-branch' # current branch +``` + +### Run Local Test + +```bash +# Make sure you're on the branch you want to test +git checkout your-feature-branch + +# Run the full validation test +./test-local.sh + +# OR test MCP connection only (faster, verifies Usable integration) +./test-local.sh --mcp-only +``` + +The full test will: +1. Verify all required environment variables are set +2. Install ForgeCode CLI if not already installed +3. Fetch the necessary git refs +4. Set up the provider authentication +5. Run the validation +6. Display the results and save artifacts to `/tmp/` + +The MCP-only test will: +1. Verify environment variables (API keys) +2. Install ForgeCode CLI if needed +3. Configure the Usable MCP server +4. Test that MCP tools are available to the AI model +5. Verify workspace access and authentication +6. Display connection results + +**Tip**: Run `--mcp-only` first to verify your Usable integration is working before running a full validation. + +### Viewing Results + +**Full validation:** +- **Full output**: `/tmp/validation-full-output.md` +- **Validation report**: `/tmp/validation-report.md` +- **GitHub outputs**: `/tmp/github-output.txt` + +**MCP test:** +- **MCP test output**: `/tmp/mcp-test-output.txt` +- **MCP test results**: `/tmp/mcp-test-result.json` + ## ๐Ÿ“– Configuration ### Inputs @@ -110,10 +179,9 @@ That's it! Your PRs will now be validated automatically. ๐ŸŽ‰ | `prompt-fragment-id` | Usable fragment UUID to use as prompt (required when `use-dynamic-prompts` is `true`) | โœ“ (with dynamic prompts) | | | `workspace-id` | Usable workspace UUID (required - used to fetch MCP system prompt) | โœ“ | | | `merge-custom-prompt` | Merge fetched Usable prompt with custom `prompt-file` (only when both are provided) | | `true` | -| `gemini-model` | Gemini model to use (`gemini-2.5-flash`, `gemini-2.0-flash`, `gemini-2.5-pro`) | | `gemini-2.5-flash` | -| `service-account-key-secret` | Secret name for service account key | | `GEMINI_SERVICE_ACCOUNT_KEY` | -| `mcp-server-url` | Usable MCP server URL | | `https://usable.dev/api/mcp` | -| `mcp-token-secret` | Secret name for Usable API token | | `USABLE_API_TOKEN` | +| `provider` | LLM provider: `openrouter`, `openai`, `anthropic`, or `auto` (auto-detect from env vars) | | `auto` | +| `model` | Model to use (e.g., `anthropic/claude-haiku-4.5`, `anthropic/claude-3.7-sonnet`, `openai/gpt-4`, `meta-llama/llama-3.3-70b-instruct`) | | `anthropic/claude-haiku-4.5` | +| `api-key-secret` | Name of secret containing API key | | `OPENROUTER_API_KEY` | | `fail-on-critical` | Fail build on critical violations | | `true` | | `comment-mode` | PR comment behavior (`update`/`new`/`none`) | | `update` | | `comment-title` | Title for PR comment (for multi-stage validation) | | `Automated Standards Validation` | @@ -124,7 +192,51 @@ That's it! Your PRs will now be validated automatically. ๐ŸŽ‰ | `head-ref` | Head reference for diff comparison | | PR head branch | | `allow-web-fetch` | Allow AI to use web_fetch tool for external resources (security consideration) | | `false` | -> **Note**: You must set the `USABLE_API_TOKEN` secret (or the custom secret name specified in `mcp-token-secret`). Usable MCP integration is required for this action. +#### Environment Variables (Required in Workflow) + +| Variable | Description | Required | +|----------|-------------|----------| +| `OPENROUTER_API_KEY` | Your OpenRouter API key (or `ANTHROPIC_API_KEY`/`OPENAI_API_KEY`) | โœ“ | +| `USABLE_API_TOKEN` | Your Usable API token for MCP integration | โœ“ | +| `USABLE_URL` | Usable instance URL (defaults to `https://usable.dev`) | | + +> **Note**: You must set the `USABLE_API_TOKEN` secret. Usable MCP integration is required for this action. + +### ๐Ÿ”— MCP Integration (Automatic) + +The action automatically configures the **Usable MCP (Model Context Protocol) server** to give the AI model access to your team's knowledge base during validation. + +**How it works:** + +1. The action uses the `@usabledev/mcp-server` package (stdio transport) +2. Runs as a subprocess via `npx @usabledev/mcp-server@latest server` +3. Authenticates using the `USABLE_API_TOKEN` environment variable +4. Provides the AI model with tools like: + - `mcp_usable_list-workspaces` - List accessible workspaces + - `mcp_usable_agentic-search-fragments` - Search knowledge base + - `mcp_usable_get-memory-fragment-content` - Read full fragments + - `mcp_usable_create-memory-fragment` - Document deviations + - And more... + +**Configuration:** + +- **No manual setup required** - the action configures everything automatically +- **Node.js required** - The MCP server requires Node.js (available in GitHub Actions by default) +- **Network access** - The MCP server connects to your Usable instance to fetch knowledge + +**Testing MCP Connection:** + +```bash +# Test locally with the --mcp-only flag +./test-local.sh --mcp-only +``` + +This will verify that: + +- โœ… MCP server starts successfully +- โœ… AI model can access MCP tools +- โœ… Workspace authentication works +- โœ… Knowledge base is accessible ### ๐Ÿง  System Prompts (Automatic) @@ -180,7 +292,7 @@ Instead of maintaining static prompt files, you can now fetch prompts dynamicall prompt-fragment-id: 'user-prompt-uuid' workspace-id: 'your-workspace-uuid' env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} # Static user prompt file (most common) @@ -189,7 +301,7 @@ Instead of maintaining static prompt files, you can now fetch prompts dynamicall prompt-file: '.github/prompts/pr-validation.md' workspace-id: 'your-workspace-uuid' env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} ``` @@ -212,22 +324,36 @@ Instead of maintaining static prompt files, you can now fetch prompts dynamicall prompt-file: '.github/prompts/validate.md' workspace-id: 'your-workspace-uuid' env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} ``` -### With Custom MCP Server +### With Anthropic Claude ```yaml - uses: flowcore/usable-pr-validator@latest with: prompt-file: '.github/prompts/validate.md' workspace-id: 'your-workspace-uuid' - mcp-server-url: 'https://your-custom-mcp.com/api/mcp' - mcp-token-secret: 'YOUR_CUSTOM_TOKEN' + provider: 'anthropic' + model: 'claude-3.7-sonnet' env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} - YOUR_CUSTOM_TOKEN: ${{ secrets.YOUR_MCP_TOKEN }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} +``` + +### With OpenAI GPT + +```yaml +- uses: flowcore/usable-pr-validator@latest + with: + prompt-file: '.github/prompts/validate.md' + workspace-id: 'your-workspace-uuid' + provider: 'openai' + model: 'gpt-4' + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} ``` ### Advanced Configuration @@ -237,17 +363,16 @@ Instead of maintaining static prompt files, you can now fetch prompts dynamicall with: prompt-file: '.github/validation/standards.md' workspace-id: 'your-workspace-uuid' - gemini-model: 'gemini-2.5-pro' - service-account-key-secret: 'MY_GEMINI_KEY' - mcp-server-url: 'https://confluence.company.com/api/mcp' - mcp-token-secret: 'CONFLUENCE_TOKEN' + provider: 'openrouter' + model: 'anthropic/claude-3.7-sonnet' + mcp-server-url: 'https://usable.dev/api/mcp' fail-on-critical: true comment-mode: 'update' artifact-retention-days: 90 max-retries: 3 env: - MY_GEMINI_KEY: ${{ secrets.GOOGLE_AI_KEY }} - CONFLUENCE_TOKEN: ${{ secrets.CONF_API_TOKEN }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} ``` ### Multiple Validation Stages @@ -269,23 +394,23 @@ jobs: workspace-id: 'your-workspace-uuid' comment-title: 'Backend Validation' # Creates unique comment env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} - + validate-frontend: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - + - uses: flowcore/usable-pr-validator@latest with: prompt-file: '.github/prompts/frontend-standards.md' workspace-id: 'your-workspace-uuid' comment-title: 'Frontend Validation' # Creates unique comment env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} ``` diff --git a/action.yml b/action.yml index 571f5e1..7919420 100644 --- a/action.yml +++ b/action.yml @@ -1,5 +1,5 @@ name: 'Usable PR Validator' -description: 'Validate Pull Requests against your Usable knowledge base standards using Google Gemini AI and Usable MCP integration' +description: 'Validate Pull Requests against your Usable knowledge base standards using ForgeCode with OpenRouter, OpenAI, Anthropic, or other LLM providers' author: 'Flowcore' branding: @@ -27,22 +27,18 @@ inputs: description: 'User comment requesting validation override or clarification (typically from @usable mention)' required: false default: '' - gemini-model: - description: 'Gemini model to use (gemini-2.5-flash, gemini-2.0-flash, gemini-2.5-pro)' + provider: + description: 'LLM provider: openrouter, openai, anthropic, or auto (auto-detect from env vars)' required: false - default: 'gemini-2.5-flash' - service-account-key-secret: - description: 'Name of secret containing base64-encoded Gemini service account key' + default: 'auto' + model: + description: 'Model to use (e.g., anthropic/claude-haiku-4.5, anthropic/claude-3.7-sonnet, openai/gpt-4, meta-llama/llama-3.3-70b-instruct)' required: false - default: 'GEMINI_SERVICE_ACCOUNT_KEY' - mcp-server-url: - description: 'HTTP URL of MCP server' + default: 'anthropic/claude-haiku-4.5' + api-key-secret: + description: 'Name of secret containing API key (OPENROUTER_API_KEY, OPENAI_API_KEY, or ANTHROPIC_API_KEY)' required: false - default: 'https://usable.dev/api/mcp' - mcp-token-secret: - description: 'Name of secret containing Usable API token' - required: false - default: 'USABLE_API_TOKEN' + default: 'OPENROUTER_API_KEY' workspace-id: description: 'Usable workspace UUID (required - used to fetch MCP system prompt)' required: true @@ -164,12 +160,12 @@ runs: with: node-version: '20' - - name: Install Gemini CLI + - name: Install ForgeCode CLI shell: bash run: | - echo "::group::Installing Gemini CLI" - npm install -g @google/gemini-cli@0.7.0 - echo "โœ… Gemini CLI installed" + echo "::group::Installing ForgeCode CLI" + npm install -g forgecode@latest + echo "โœ… ForgeCode CLI installed" echo "::endgroup::" - name: Setup Git @@ -276,18 +272,19 @@ runs: echo "โœ… Git configured" echo "::endgroup::" - - name: Setup Gemini Authentication + - name: Setup LLM Provider Authentication shell: bash env: - SECRET_NAME: ${{ inputs.service-account-key-secret }} + PROVIDER: ${{ inputs.provider }} + API_KEY_SECRET: ${{ inputs.api-key-secret }} run: | - ${{ github.action_path }}/scripts/setup-gemini.sh + ${{ github.action_path }}/scripts/setup-provider.sh - name: Setup MCP Server shell: bash env: - MCP_URL: ${{ inputs.mcp-server-url }} - MCP_SECRET_NAME: ${{ inputs.mcp-token-secret }} + USABLE_URL: ${{ env.USABLE_URL || 'https://usable.dev' }} + USABLE_API_TOKEN: ${{ env.USABLE_API_TOKEN }} WORKSPACE_ID: ${{ inputs.workspace-id }} run: | ${{ github.action_path }}/scripts/setup-mcp.sh @@ -300,7 +297,8 @@ runs: WORKSPACE_ID: ${{ inputs.workspace-id }} CUSTOM_PROMPT_FILE: ${{ inputs.prompt-file }} MERGE_CUSTOM_PROMPT: ${{ inputs.merge-custom-prompt }} - MCP_SECRET_NAME: ${{ inputs.mcp-token-secret }} + USABLE_URL: ${{ env.USABLE_URL || 'https://usable.dev' }} + USABLE_API_TOKEN: ${{ env.USABLE_API_TOKEN }} ACTION_PATH: ${{ github.action_path }} run: | ${{ github.action_path }}/scripts/fetch-prompt.sh @@ -311,7 +309,8 @@ runs: env: PROMPT_FILE: ${{ inputs.prompt-file }} USE_DYNAMIC_PROMPTS: ${{ inputs.use-dynamic-prompts }} - GEMINI_MODEL: ${{ inputs.gemini-model }} + PROVIDER: ${{ inputs.provider }} + MODEL: ${{ inputs.model }} MAX_RETRIES: ${{ inputs.max-retries }} PR_NUMBER: ${{ github.event.pull_request.number }} PR_TITLE: ${{ github.event.pull_request.title }} @@ -326,7 +325,98 @@ runs: ALLOW_WEB_FETCH: ${{ inputs.allow-web-fetch }} GIT_PAGER: cat run: | - ${{ github.action_path }}/scripts/validate.sh + echo "::group::Running PR Validation" + + # Determine which prompt file to use + if [ -f "/tmp/dynamic-prompt.md" ]; then + ACTUAL_PROMPT="/tmp/dynamic-prompt.md" + echo "Using dynamic prompt from fetch-prompt.sh" + elif [ -n "$PROMPT_FILE" ] && [ -f "$PROMPT_FILE" ]; then + ACTUAL_PROMPT="$PROMPT_FILE" + echo "Using static prompt file: $PROMPT_FILE" + else + echo "::error::No valid prompt file found" + exit 1 + fi + + # Configure ForgeCode model if specified + if [ -n "$MODEL" ]; then + echo "Configuring ForgeCode model: $MODEL" + forge config set --model "$MODEL" || echo "Warning: Could not set model config" + fi + + # Refresh MCP cache before validation + echo "๐Ÿ”„ Refreshing MCP cache..." + set +e + CACHE_OUTPUT=$(timeout 30 forge mcp cache refresh 2>&1) + CACHE_EXIT=$? + set -e + + if [ $CACHE_EXIT -eq 0 ]; then + echo "โœ… MCP cache refreshed successfully" + elif [ $CACHE_EXIT -eq 124 ]; then + echo "โš ๏ธ MCP cache refresh timed out after 30 seconds (continuing anyway)" + elif echo "$CACHE_OUTPUT" | grep -q "No such file or directory"; then + echo "โš ๏ธ No cache directory exists yet (first run) - skipping cache refresh" + else + echo "โš ๏ธ MCP cache refresh failed - continuing anyway" + echo " Error: $(echo "$CACHE_OUTPUT" | grep -i "error" | head -1)" + fi + echo "" + + # Run ForgeCode directly with MCP stdio transport + echo "๐Ÿค– Running ForgeCode CLI with MCP stdio transport" + echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" + echo "Provider: ${PROVIDER:-auto}" + echo "Model: $MODEL" + echo "Prompt: $ACTUAL_PROMPT" + echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" + + # Run forge and capture output + set +e + forge -p "$(cat "$ACTUAL_PROMPT")" 2>&1 | tee /tmp/validation-full-output.md + FORGE_EXIT=$? + set -e + + echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" + + if [ $FORGE_EXIT -ne 0 ]; then + echo "::error::ForgeCode failed with exit code: $FORGE_EXIT" + echo "validation_status=error" >> $GITHUB_OUTPUT + echo "validation_passed=false" >> $GITHUB_OUTPUT + echo "critical_issues=0" >> $GITHUB_OUTPUT + exit 1 + fi + + # Process the JSON report if it exists + if [ -f "/tmp/validation-report.json" ]; then + # Construct markdown report + ${{ github.action_path }}/scripts/construct-markdown.sh /tmp/validation-report.json /tmp/validation-report.md + + # Parse results + STATUS=$(jq -r '.validationOutcome.status' /tmp/validation-report.json) + CRITICAL=$(jq -r '.validationOutcome.criticalIssuesCount' /tmp/validation-report.json) + + if [ "$STATUS" = "PASS" ]; then + echo "validation_status=passed" >> $GITHUB_OUTPUT + echo "validation_passed=true" >> $GITHUB_OUTPUT + else + echo "validation_status=failed" >> $GITHUB_OUTPUT + echo "validation_passed=false" >> $GITHUB_OUTPUT + fi + + echo "critical_issues=${CRITICAL:-0}" >> $GITHUB_OUTPUT + + echo "โœ… Validation completed: $STATUS (Critical: ${CRITICAL:-0})" + else + echo "::warning::No JSON report generated" + cp /tmp/validation-full-output.md /tmp/validation-report.md || true + echo "validation_status=completed" >> $GITHUB_OUTPUT + echo "validation_passed=true" >> $GITHUB_OUTPUT + echo "critical_issues=0" >> $GITHUB_OUTPUT + fi + + echo "::endgroup::" - name: Post PR Comment if: inputs.comment-mode != 'none' && always() @@ -349,9 +439,10 @@ runs: const markerId = commentTitle.toLowerCase().replace(/[^a-z0-9]+/g, '-'); const marker = ``; - const commentBody = marker + '\n## ๐Ÿค– ' + commentTitle + '\n\n' + + const commentBody = marker + '\n## ๐Ÿค– ' + commentTitle + '\n\n' + report + '\n\n---\n
\n๐Ÿ“Š Validation Statistics\n\n' + - '- **Model**: Google Gemini ${{ inputs.gemini-model }}\n' + + '- **Provider**: ${{ inputs.provider }}\n' + + '- **Model**: ${{ inputs.model }}\n' + '- **Standards Source**: ${{ inputs.mcp-server-url }}\n' + '- **Commit**: ' + context.payload.pull_request.head.sha.substring(0, 7) + '\n' + '- **Triggered by**: @' + context.actor + '\n\n' + @@ -408,10 +499,13 @@ runs: shell: bash run: | # Remove temporary files containing sensitive data - rm -f /tmp/service-account.json - rm -f /tmp/gemini-settings.json - rm -f /tmp/gemini-stderr.log - echo "โœ… Temporary files cleaned up" + rm -f /tmp/forge-config.yaml + rm -f /tmp/validation-*.md + + # Remove MCP server registration (contains authentication token) + forge mcp remove usable 2>/dev/null || true + + echo "โœ… Temporary files and MCP registration cleaned up" - name: Fail on Critical Violations if: inputs.fail-on-critical == 'true' && steps.validate.outputs.validation_passed == 'false' diff --git a/forge.yaml b/forge.yaml new file mode 100644 index 0000000..cc03a6a --- /dev/null +++ b/forge.yaml @@ -0,0 +1,10 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/antinomyhq/forge/refs/heads/main/forge.schema.json +mcpServers: + usable: + command: npx + args: + - "@usabledev/mcp-server@latest" + - server + env: + USABLE_API_TOKEN: ${USABLE_API_TOKEN} + USABLE_BASE_URL: ${USABLE_URL} diff --git a/schemas/validation-report.schema.json b/schemas/validation-report.schema.json new file mode 100644 index 0000000..95c9b31 --- /dev/null +++ b/schemas/validation-report.schema.json @@ -0,0 +1,169 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "PR Validation Report", + "description": "Structured output schema for PR validation results", + "type": "object", + "required": ["summary", "criticalViolations", "importantIssues", "suggestions", "validationOutcome"], + "properties": { + "summary": { + "type": "string", + "description": "Brief 2-3 sentence overview of the PR and overall assessment" + }, + "criticalViolations": { + "type": "array", + "description": "Must-fix issues that will fail the build", + "items": { + "type": "object", + "required": ["file", "issue", "why", "fix"], + "properties": { + "file": { + "type": "string", + "description": "File path and optional line number (e.g., 'path/to/file.ts:42')" + }, + "issue": { + "type": "string", + "description": "Clear description of the violation" + }, + "why": { + "type": "string", + "description": "Explanation of why this violates standards" + }, + "fix": { + "type": "string", + "description": "Specific recommendation on how to fix it" + }, + "code": { + "type": "string", + "description": "Optional: The problematic code snippet" + } + } + } + }, + "importantIssues": { + "type": "array", + "description": "Should-fix issues that won't fail the build", + "items": { + "type": "object", + "required": ["file", "issue", "why", "fix"], + "properties": { + "file": { + "type": "string", + "description": "File path and optional line number" + }, + "issue": { + "type": "string", + "description": "Clear description of the issue" + }, + "why": { + "type": "string", + "description": "Explanation of the issue" + }, + "fix": { + "type": "string", + "description": "Recommended improvement" + }, + "code": { + "type": "string", + "description": "Optional: The problematic code snippet" + } + } + } + }, + "suggestions": { + "type": "array", + "description": "Nice-to-have improvements", + "items": { + "type": "object", + "required": ["title", "description"], + "properties": { + "title": { + "type": "string", + "description": "Brief title of the suggestion" + }, + "description": { + "type": "string", + "description": "Detailed explanation of the suggestion" + }, + "file": { + "type": "string", + "description": "Optional: Related file path" + } + } + } + }, + "validationOutcome": { + "type": "object", + "required": ["status", "criticalIssuesCount", "importantIssuesCount", "suggestionsCount"], + "properties": { + "status": { + "type": "string", + "enum": ["PASS", "FAIL"], + "description": "Overall validation status" + }, + "criticalIssuesCount": { + "type": "integer", + "minimum": 0, + "description": "Number of critical violations found" + }, + "importantIssuesCount": { + "type": "integer", + "minimum": 0, + "description": "Number of important issues found" + }, + "suggestionsCount": { + "type": "integer", + "minimum": 0, + "description": "Number of suggestions provided" + }, + "rationale": { + "type": "string", + "description": "Optional: Brief explanation of the validation outcome" + } + } + }, + "overrideApplied": { + "type": "object", + "description": "Optional: Information about applied overrides (for comment-triggered revalidation)", + "properties": { + "deviation": { + "type": "string", + "description": "What deviation was approved" + }, + "justification": { + "type": "string", + "description": "Why the deviation was necessary" + }, + "fragmentId": { + "type": "string", + "description": "ID of the created deviation fragment" + }, + "fragmentTitle": { + "type": "string", + "description": "Title of the created deviation fragment" + }, + "approvedBy": { + "type": "string", + "description": "Username who approved the deviation" + } + } + }, + "metadata": { + "type": "object", + "description": "Optional: Additional metadata about the validation", + "properties": { + "triggeredBy": { + "type": "string", + "description": "How this validation was triggered (e.g., 'PR opened', 'Comment by @user')" + }, + "standardsChecked": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of standards/patterns checked" + } + } + } + } +} + diff --git a/scripts/construct-markdown.sh b/scripts/construct-markdown.sh new file mode 100755 index 0000000..46206db --- /dev/null +++ b/scripts/construct-markdown.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Construct markdown comment from JSON validation report +# Usage: construct-markdown.sh + +JSON_FILE="${1:-/tmp/validation-report.json}" +OUTPUT_FILE="${2:-/tmp/validation-report.md}" + +if [ ! -f "$JSON_FILE" ]; then + echo "::error::JSON report file not found: $JSON_FILE" + exit 1 +fi + +# Verify JSON is valid +if ! jq empty "$JSON_FILE" 2>/dev/null; then + echo "::error::Invalid JSON in report file" + exit 1 +fi + +# Start constructing the markdown report +{ + echo "# PR Validation Report" + echo "" + + # Summary + echo "## Summary" + jq -r '.summary' "$JSON_FILE" + echo "" + + # Override information (if present) + if jq -e '.overrideApplied' "$JSON_FILE" > /dev/null 2>&1; then + echo "## Override Applied ๐Ÿ”„" + echo "" + echo "A deviation from standards has been approved and documented:" + echo "" + echo "- **Deviation**: $(jq -r '.overrideApplied.deviation // "N/A"' "$JSON_FILE")" + echo "- **Justification**: $(jq -r '.overrideApplied.justification // "N/A"' "$JSON_FILE")" + + if jq -e '.overrideApplied.fragmentId' "$JSON_FILE" > /dev/null 2>&1; then + fragment_id=$(jq -r '.overrideApplied.fragmentId' "$JSON_FILE") + fragment_title=$(jq -r '.overrideApplied.fragmentTitle // "Deviation"' "$JSON_FILE") + echo "- **Documentation**: Fragment created - ${fragment_title} (ID: ${fragment_id})" + fi + + if jq -e '.overrideApplied.approvedBy' "$JSON_FILE" > /dev/null 2>&1; then + approved_by=$(jq -r '.overrideApplied.approvedBy' "$JSON_FILE") + echo "- **Approved by**: @${approved_by}" + fi + + echo "" + echo "This deviation has been recorded in the knowledge base for future reference." + echo "" + fi + + # Critical Violations + echo "## Critical Violations โŒ" + echo "" + + critical_count=$(jq '.criticalViolations | length' "$JSON_FILE") + + if [ "$critical_count" -eq 0 ]; then + echo "None found." + else + # Iterate through critical violations + jq -c '.criticalViolations[]' "$JSON_FILE" | while read -r violation; do + file=$(echo "$violation" | jq -r '.file') + issue=$(echo "$violation" | jq -r '.issue') + why=$(echo "$violation" | jq -r '.why') + fix=$(echo "$violation" | jq -r '.fix') + code=$(echo "$violation" | jq -r '.code // empty') + + echo "- **File**: \`$file\`" + echo " - **Issue**: $issue" + echo " - **Why**: $why" + echo " - **Fix**: $fix" + + if [ -n "$code" ]; then + echo " - **Code**:" + echo ' ```' + echo "$code" | sed 's/^/ /' + echo ' ```' + fi + echo "" + done + fi + + echo "" + + # Important Issues + echo "## Important Issues โš ๏ธ" + echo "" + + important_count=$(jq '.importantIssues | length' "$JSON_FILE") + + if [ "$important_count" -eq 0 ]; then + echo "None found." + else + jq -c '.importantIssues[]' "$JSON_FILE" | while read -r issue; do + file=$(echo "$issue" | jq -r '.file') + issue_text=$(echo "$issue" | jq -r '.issue') + why=$(echo "$issue" | jq -r '.why') + fix=$(echo "$issue" | jq -r '.fix') + code=$(echo "$issue" | jq -r '.code // empty') + + echo "- **File**: \`$file\`" + echo " - **Issue**: $issue_text" + echo " - **Why**: $why" + echo " - **Fix**: $fix" + + if [ -n "$code" ]; then + echo " - **Code**:" + echo ' ```' + echo "$code" | sed 's/^/ /' + echo ' ```' + fi + echo "" + done + fi + + echo "" + + # Suggestions + echo "## Suggestions โ„น๏ธ" + echo "" + + suggestions_count=$(jq '.suggestions | length' "$JSON_FILE") + + if [ "$suggestions_count" -eq 0 ]; then + echo "None found." + else + jq -c '.suggestions[]' "$JSON_FILE" | while read -r suggestion; do + title=$(echo "$suggestion" | jq -r '.title') + description=$(echo "$suggestion" | jq -r '.description') + file=$(echo "$suggestion" | jq -r '.file // empty') + + echo "- **$title**: $description" + + if [ -n "$file" ]; then + echo " - File: \`$file\`" + fi + echo "" + done + fi + + echo "" + + # Validation Outcome + echo "## Validation Outcome" + echo "" + + status=$(jq -r '.validationOutcome.status' "$JSON_FILE") + critical=$(jq -r '.validationOutcome.criticalIssuesCount' "$JSON_FILE") + important=$(jq -r '.validationOutcome.importantIssuesCount' "$JSON_FILE") + suggestions=$(jq -r '.validationOutcome.suggestionsCount' "$JSON_FILE") + + if [ "$status" = "PASS" ]; then + echo "- **Status**: PASS โœ…" + else + echo "- **Status**: FAIL โŒ" + fi + + echo "- **Critical Issues**: $critical" + echo "- **Important Issues**: $important" + echo "- **Suggestions**: $suggestions" + + # Add rationale if present + if jq -e '.validationOutcome.rationale' "$JSON_FILE" > /dev/null 2>&1; then + rationale=$(jq -r '.validationOutcome.rationale' "$JSON_FILE") + echo "" + echo "**Rationale**: $rationale" + fi + + # Add metadata if present + if jq -e '.metadata.triggeredBy' "$JSON_FILE" > /dev/null 2>&1; then + triggered_by=$(jq -r '.metadata.triggeredBy' "$JSON_FILE") + echo "" + echo "- **Triggered by**: $triggered_by" + fi + + if jq -e '.metadata.standardsChecked' "$JSON_FILE" > /dev/null 2>&1; then + echo "" + echo "### Standards Checked" + echo "" + jq -r '.metadata.standardsChecked[]' "$JSON_FILE" | while read -r standard; do + echo "- $standard" + done + fi + +} > "$OUTPUT_FILE" + +echo "โœ… Markdown report constructed: $OUTPUT_FILE" +echo " Lines: $(wc -l < "$OUTPUT_FILE")" + diff --git a/scripts/fetch-prompt.sh b/scripts/fetch-prompt.sh index b3f25f7..07a3f87 100755 --- a/scripts/fetch-prompt.sh +++ b/scripts/fetch-prompt.sh @@ -9,16 +9,21 @@ if ! command -v jq &> /dev/null; then exit 1 fi -# Get USABLE_API_TOKEN from secrets -USABLE_API_TOKEN="${!MCP_SECRET_NAME:-}" -if [ -z "$USABLE_API_TOKEN" ]; then +# Get USABLE_API_TOKEN from environment +# The token should be passed directly as USABLE_API_TOKEN env var +if [ -z "${USABLE_API_TOKEN:-}" ]; then echo "::warning::USABLE_API_TOKEN not found. Skipping MCP system prompt fetching." HAS_API_TOKEN=false else HAS_API_TOKEN=true + echo "โœ… USABLE_API_TOKEN found" fi -USABLE_API_BASE="https://usable.dev/api" +# Get Usable URL from environment (default to usable.dev) +USABLE_URL="${USABLE_URL:-https://usable.dev}" +USABLE_API_BASE="${USABLE_URL}/api" +echo "Usable API Base: $USABLE_API_BASE" + HARDCODED_SYSTEM_PROMPT="${ACTION_PATH}/system-prompt.md" MCP_SYSTEM_PROMPT_FILE="/tmp/mcp-system-prompt.md" USER_PROMPT_FILE="/tmp/user-prompt.md" @@ -90,20 +95,15 @@ fetch_mcp_system_prompt() { return 1 fi - # The API might return JSON with a content field, or plain text - # Try to parse as JSON first using jq - # Note: jq is pre-installed on GitHub Actions runners + # The API returns JSON with a systemPrompt field + # Extract it using jq local content - content=$(echo "$body" | jq -r '.content // .prompt // empty' 2>/dev/null) - - # If JSON parsing returned empty, use the body as-is (assuming plain text) - if [ -z "$content" ]; then - content="$body" - fi + content=$(echo "$body" | jq -r '.systemPrompt // empty' 2>/dev/null) # Verify content is not empty after parsing if [ -z "$content" ]; then echo "::warning::MCP system prompt content is empty after parsing." + echo "::debug::Response body: $body" return 1 fi @@ -169,15 +169,15 @@ main() { fi else # Static prompt file - if [ -n "$CUSTOM_PROMPT_FILE" ] && [ -f "$CUSTOM_PROMPT_FILE" ]; then + if [ -n "${CUSTOM_PROMPT_FILE:-}" ] && [ -f "$CUSTOM_PROMPT_FILE" ]; then echo "Using static prompt file: $CUSTOM_PROMPT_FILE" cp "$CUSTOM_PROMPT_FILE" "$USER_PROMPT_FILE" has_user_prompt=true echo "โœ… Static prompt loaded" echo "Size: $(wc -c < "$USER_PROMPT_FILE") bytes" else - echo "::error::No user prompt file provided or file not found" - exit 1 + echo "โ„น๏ธ No custom user prompt file specified - using only hardcoded system + MCP system prompts" + has_user_prompt=false fi fi diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index d2a6580..11d371c 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -3,48 +3,56 @@ set -euo pipefail echo "::group::Setting up MCP Server Integration" -# Get the MCP token from environment using the secret name -MCP_TOKEN="${!MCP_SECRET_NAME:-}" - -if [ -z "$MCP_TOKEN" ]; then - echo "::error::MCP token not found in environment variable: $MCP_SECRET_NAME" - echo "Please ensure the secret is set in your workflow: env.$MCP_SECRET_NAME" +# Get the Usable API token from environment +if [ -z "${USABLE_API_TOKEN:-}" ]; then + echo "::error::USABLE_API_TOKEN not found" + echo "Please set USABLE_API_TOKEN environment variable or secret" exit 1 fi -# Validate MCP URL -if [ -z "$MCP_URL" ]; then - echo "::error::MCP_URL is required when MCP is enabled" - exit 1 -fi +# Get Usable URL from environment (default to usable.dev) +USABLE_URL="${USABLE_URL:-https://usable.dev}" -# Create Gemini settings file with MCP configuration -cat > /tmp/gemini-settings.json <> $GITHUB_ENV - -echo "โœ… MCP server configured" -echo " URL: $MCP_URL" -echo " Settings file: /tmp/gemini-settings.json" - -# Debug: Show settings file content (mask token) -echo " Configuration preview:" -cat /tmp/gemini-settings.json | sed 's/"Bearer [^"]*"/"Bearer ***MASKED***"/g' | sed 's/^/ /' +) + +echo " Type: stdio transport via npx" +echo " Command: npx @usabledev/mcp-server@latest server" +echo " Base URL: ${USABLE_URL}" +echo "" + +# Remove existing server if it exists (to avoid conflicts) +forge mcp remove usable 2>/dev/null || true + +# Register the server +if forge mcp add-json usable "$SERVER_CONFIG" --scope local 2>&1; then + echo "โœ… MCP server 'usable' registered successfully" + echo "" + echo "Verifying registration..." + forge mcp list 2>&1 | grep -i "usable" || echo "โš ๏ธ Server registered but not shown in list" +else + echo "::error::Failed to register MCP server with ForgeCode" + exit 1 +fi echo "::endgroup::" diff --git a/scripts/setup-provider.sh b/scripts/setup-provider.sh new file mode 100755 index 0000000..363ae93 --- /dev/null +++ b/scripts/setup-provider.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "::group::Setting up LLM Provider Authentication" + +# Determine which provider to use +PROVIDER="${PROVIDER:-auto}" + +echo "Provider mode: $PROVIDER" + +# Function to check if an API key is available +check_api_key() { + local key_name="$1" + local key_value="${!key_name:-}" + + if [ -n "$key_value" ]; then + echo "โœ… $key_name is available" + return 0 + else + echo "โŒ $key_name is not set" + return 1 + fi +} + +# Auto-detect provider if set to auto +if [ "$PROVIDER" = "auto" ]; then + echo "Auto-detecting provider from environment variables..." + + if check_api_key "OPENROUTER_API_KEY"; then + PROVIDER="openrouter" + echo "๐ŸŽฏ Auto-detected provider: OpenRouter" + elif check_api_key "ANTHROPIC_API_KEY"; then + PROVIDER="anthropic" + echo "๐ŸŽฏ Auto-detected provider: Anthropic" + elif check_api_key "OPENAI_API_KEY"; then + PROVIDER="openai" + echo "๐ŸŽฏ Auto-detected provider: OpenAI" + else + echo "::error::No API key found in environment" + echo "Please set one of: OPENROUTER_API_KEY, ANTHROPIC_API_KEY, OPENAI_API_KEY" + exit 1 + fi +fi + +# Validate that the required API key is present +case "$PROVIDER" in + openrouter) + if ! check_api_key "OPENROUTER_API_KEY"; then + echo "::error::OPENROUTER_API_KEY is required for OpenRouter provider" + exit 1 + fi + ;; + anthropic) + if ! check_api_key "ANTHROPIC_API_KEY"; then + echo "::error::ANTHROPIC_API_KEY is required for Anthropic provider" + exit 1 + fi + ;; + openai) + if ! check_api_key "OPENAI_API_KEY"; then + echo "::error::OPENAI_API_KEY is required for OpenAI provider" + exit 1 + fi + ;; + *) + echo "::error::Unknown provider: $PROVIDER" + echo "Supported providers: openrouter, anthropic, openai, auto" + exit 1 + ;; +esac + +# Export provider for downstream scripts +echo "PROVIDER=$PROVIDER" >> "$GITHUB_ENV" 2>/dev/null || export PROVIDER="$PROVIDER" + +echo "โœ… Provider authentication configured: $PROVIDER" +echo "::endgroup::" diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh new file mode 100755 index 0000000..19da524 --- /dev/null +++ b/scripts/test-mcp-connection.sh @@ -0,0 +1,227 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "::group::Testing MCP Connection" + +# Check if MCP server is registered with ForgeCode +echo "Checking registered MCP servers..." +if ! forge mcp list 2>&1 | grep -q "usable"; then + echo "::error::MCP server 'usable' not found in ForgeCode configuration" + echo "Available servers:" + forge mcp list 2>&1 || echo " (none)" + echo "" + echo "Please run setup-mcp.sh first to register the MCP server" + exit 1 +fi + +echo "โœ… MCP server 'usable' is registered" +echo "" +echo "Configuration:" +forge mcp get usable 2>&1 || echo " (Unable to retrieve configuration)" + +echo "" +echo "Creating MCP test prompt..." + +# Setup provider configuration first (same as validate.sh) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +echo "" +echo "Setting up provider configuration..." +if [ -f "$SCRIPT_DIR/setup-provider.sh" ]; then + source "$SCRIPT_DIR/setup-provider.sh" || { + echo "::error::Failed to setup provider" + exit 1 + } +else + echo "::warning::setup-provider.sh not found, skipping provider setup" +fi + +echo "" + +# Set ForgeCode model configuration (prevents interactive prompt) +if [ -n "${MODEL:-}" ]; then + echo "Configuring ForgeCode model: ${MODEL}" + forge config set --model "${MODEL}" 2>&1 || echo "::warning::Could not set model config" +else + echo "::warning::MODEL not set, ForgeCode may prompt for model selection" +fi + +echo "" +# Create a simple test prompt that lists available tools +cat > /tmp/mcp-test-prompt.txt <<'EOF' +# MCP Connection Test + +Your task is to verify the MCP connection by listing your available tools. + +## Instructions + +1. List ALL available tools you have access to +2. Look for tools that start with `mcp_usable_` prefix +3. If you find MCP tools, call `mcp_usable_list-workspaces` with parameter `{"includeArchived": false}` +4. Report the results + +## Output Format + +Write your findings to `/tmp/mcp-test-result.json` in this format: + +```json +{ + "mcpToolsFound": true/false, + "toolCount": 0, + "mcpTools": ["list", "of", "mcp", "tools"], + "workspacesResult": { + "success": true/false, + "workspaces": [], + "error": "error message if failed" + } +} +``` + +IMPORTANT: You must write the JSON file even if no MCP tools are found. +EOF + +echo "" +echo "Checking if ForgeCode can see MCP servers in config..." +forge mcp list 2>&1 || echo "โš ๏ธ forge mcp list command not available or failed" + +echo "" +echo "๐Ÿ”„ Refreshing MCP cache..." +set +e +CACHE_OUTPUT=$(timeout 30 forge mcp cache refresh 2>&1) +CACHE_EXIT=$? +set -e + +if [ $CACHE_EXIT -eq 0 ]; then + echo "โœ… MCP cache refreshed successfully" +elif [ $CACHE_EXIT -eq 124 ]; then + echo "โš ๏ธ MCP cache refresh timed out after 30 seconds (continuing anyway)" +elif echo "$CACHE_OUTPUT" | grep -q "No such file or directory"; then + echo "โš ๏ธ No cache directory exists yet (first run) - skipping cache refresh" +else + echo "โš ๏ธ MCP cache refresh failed - continuing anyway" + echo " Error: $(echo "$CACHE_OUTPUT" | grep -i "error" | head -1)" +fi + +echo "" +echo "Note: Using stdio transport via npx @usabledev/mcp-server" +echo " Direct subprocess communication for better control" +echo " Authentication passed via USABLE_API_TOKEN environment variable" + +echo "" +echo "Running ForgeCode with MCP test prompt..." +echo "Current working directory: $(pwd)" +echo "" +echo "Capturing both stdout and stderr to see MCP initialization..." + +# Run ForgeCode with the test prompt, capturing all output including stderr +# Use --verbose flag for detailed output +echo "Running ForgeCode with --verbose flag..." +echo "Waiting 5 seconds before starting ForgeCode to ensure MCP server registration is settled..." +sleep 5 +forge --verbose -p "$(cat /tmp/mcp-test-prompt.txt)" 2>&1 | tee /tmp/mcp-test-output.txt & +FORGE_PID=$! + +# Wait for ForgeCode to initialize and connect to MCP server via stdio +echo "Waiting for ForgeCode to initialize MCP stdio connection..." +sleep 5 + +# Check for any MCP-related patterns in the output +echo "" +echo "Checking for MCP-related logs in output..." +if [ -f /tmp/mcp-test-output.txt ] && [ -s /tmp/mcp-test-output.txt ]; then + if grep -qi "mcp\|server\|http\|initialize\|connect" /tmp/mcp-test-output.txt; then + echo "::group::MCP-related logs found" + grep -i "mcp\|server\|http\|initialize\|connect" /tmp/mcp-test-output.txt | head -50 + echo "::endgroup::" + else + echo "No MCP-related patterns found in output" + fi +else + echo "Output file is empty or doesn't exist yet" +fi + +# Wait for ForgeCode to complete +if wait $FORGE_PID; then + echo "" + echo "ForgeCode execution completed" +else + echo "" + echo "::warning::ForgeCode execution had errors" +fi + +echo "" +echo "::group::ForgeCode Output" +if [ -f /tmp/mcp-test-output.txt ]; then + echo "Combined output (first 100 lines):" + head -100 /tmp/mcp-test-output.txt +else + echo "No output captured" +fi +echo "::endgroup::" + +echo "" +echo "::group::ForgeCode Verbose Output (with --verbose flag)" +if [ -f /tmp/mcp-test-output.txt ] && [ -s /tmp/mcp-test-output.txt ]; then + echo "Verbose output (first 300 lines - may show MCP initialization details):" + tail -n +2 /tmp/mcp-test-output.txt | head -300 # Skip first line (duplicate) +else + echo "No verbose output captured" +fi +echo "::endgroup::" + +# Check if result file was created +if [ -f "/tmp/mcp-test-result.json" ]; then + echo "" + echo "::group::MCP Test Results" + cat /tmp/mcp-test-result.json + echo "" + echo "::endgroup::" + + # Parse results + if jq -e '.mcpToolsFound == true' /tmp/mcp-test-result.json > /dev/null 2>&1; then + echo "โœ… MCP tools are available to the AI model" + + # Check workspace test + if jq -e '.workspacesResult.success == true' /tmp/mcp-test-result.json > /dev/null 2>&1; then + WORKSPACE_COUNT=$(jq -r '.workspacesResult.workspaces | length' /tmp/mcp-test-result.json) + echo "โœ… MCP connection successful - found $WORKSPACE_COUNT workspace(s)" + + # List workspaces + echo "" + echo "Workspaces:" + jq -r '.workspacesResult.workspaces[] | " - \(.name) (ID: \(.id))"' /tmp/mcp-test-result.json || echo " (Unable to parse workspace list)" + + echo "::endgroup::" + exit 0 + else + echo "โŒ MCP tools found but workspace query failed" + jq -r '.workspacesResult.error // "Unknown error"' /tmp/mcp-test-result.json + echo "::endgroup::" + exit 1 + fi + else + echo "โŒ MCP tools NOT available to the AI model" + echo "" + echo "Available tools:" + jq -r '.mcpTools[]' /tmp/mcp-test-result.json || echo " (No tools information)" + echo "::endgroup::" + exit 1 + fi +else + echo "" + echo "::warning::AI model did not create result file" + echo "This likely means the model couldn't understand the prompt or write the file" + echo "" + echo "Checking output for MCP tool mentions..." + + if grep -q "mcp_usable" /tmp/mcp-test-output.txt; then + echo "โœ… Found references to mcp_usable tools in output" + else + echo "โŒ No references to mcp_usable tools found" + echo "::error::MCP tools are NOT available to the AI model" + fi + + echo "::endgroup::" + exit 1 +fi + + diff --git a/scripts/test-mcp-protocol.sh b/scripts/test-mcp-protocol.sh new file mode 100755 index 0000000..2395088 --- /dev/null +++ b/scripts/test-mcp-protocol.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Test if the MCP server responds to MCP protocol requests over stdio + +echo "Testing MCP server stdio protocol..." +echo "" + +# Export environment variables +export USABLE_API_TOKEN="${USABLE_API_TOKEN}" +export USABLE_BASE_URL="${USABLE_URL:-https://usable.dev}" + +echo "Starting MCP server and sending initialize request..." +echo "" + +# Send an MCP initialize request via stdin +echo '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"test","version":"1.0"}}}' | \ + npx --yes @usabledev/mcp-server@latest server 2>&1 | head -50 + +echo "" +echo "If you see a valid JSON-RPC response above with server info, the MCP server works." +echo "If you see errors or no response, the MCP server has issues." + diff --git a/scripts/validate.sh b/scripts/validate.sh index 9a689e2..6ad7d27 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -100,11 +100,53 @@ verify_git_refs() { fi } +# Get file stats for the PR +get_file_stats() { + local base="${BASE_BRANCH}" + local head="${HEAD_BRANCH}" + + echo "Getting file statistics for PR..." + + # Try to get the diff stat + local file_stats="" + if git diff --stat "origin/$base...origin/$head" 2>/dev/null; then + file_stats=$(git diff --stat "origin/$base...origin/$head" 2>/dev/null || echo "Unable to retrieve file statistics") + elif git diff --stat "$base...$head" 2>/dev/null; then + file_stats=$(git diff --stat "$base...$head" 2>/dev/null || echo "Unable to retrieve file statistics") + else + file_stats="Unable to retrieve file statistics. Use git commands to explore changes." + fi + + # Also get list of changed files + local changed_files="" + if git diff --name-only "origin/$base...origin/$head" 2>/dev/null; then + changed_files=$(git diff --name-only "origin/$base...origin/$head" 2>/dev/null | head -100) + elif git diff --name-only "$base...$head" 2>/dev/null; then + changed_files=$(git diff --name-only "$base...$head" 2>/dev/null | head -100) + fi + + # Format the output + local output="## Files Changed + +\`\`\` +$file_stats +\`\`\` + +## Changed Files List +\`\`\` +$changed_files +\`\`\` + +**Note**: Use \`git --no-pager diff origin/$base...origin/$head -- \` to examine specific files." + + echo "$output" +} + # Prepare prompt with placeholder replacement -# +# # Uses bash native string replacement for simplicity and safety. # For current scope (8 placeholders), this is efficient and readable. -# +# # Alternative approaches considered: # - envsubst: Would require careful escaping of $ symbols in prompts # - sed: More complex escaping, harder to maintain @@ -117,7 +159,7 @@ verify_git_refs() { prepare_prompt() { local prompt_file="$1" local output_file="/tmp/validation-prompt.txt" - + # Create PR context block PR_CONTEXT="**PR #${PR_NUMBER}**: ${PR_TITLE} @@ -129,7 +171,7 @@ prepare_prompt() { ${PR_DESCRIPTION:-No description provided}" # Add override comment if provided - if [ -n "$OVERRIDE_COMMENT" ]; then + if [ -n "${OVERRIDE_COMMENT:-}" ]; then PR_CONTEXT="${PR_CONTEXT} **๐Ÿ”„ Override/Clarification Comment** (from @${COMMENT_AUTHOR:-unknown}): @@ -147,13 +189,18 @@ ${OVERRIDE_COMMENT} web_fetch_policy="**Web fetch is DISABLED** for this validation. DO NOT use the \`web_fetch\` tool or attempt to download content from URLs. All validation must be performed using only the git repository contents, PR context, and Usable MCP knowledge base." fi + # Get file statistics + local file_stats + file_stats=$(get_file_stats) + # Read prompt template PROMPT_CONTENT=$(cat "$prompt_file") - + # Replace placeholders using bash string replacement (NOT sed) # This handles special characters safely PROMPT_CONTENT="${PROMPT_CONTENT//\{\{WEB_FETCH_POLICY\}\}/${web_fetch_policy}}" PROMPT_CONTENT="${PROMPT_CONTENT//\{\{PR_CONTEXT\}\}/${PR_CONTEXT}}" + PROMPT_CONTENT="${PROMPT_CONTENT//\{\{FILE_STATS\}\}/${file_stats}}" PROMPT_CONTENT="${PROMPT_CONTENT//\{\{BASE_BRANCH\}\}/${BASE_BRANCH}}" PROMPT_CONTENT="${PROMPT_CONTENT//\{\{HEAD_BRANCH\}\}/${HEAD_BRANCH}}" PROMPT_CONTENT="${PROMPT_CONTENT//\{\{PR_TITLE\}\}/${PR_TITLE}}" @@ -162,10 +209,10 @@ ${OVERRIDE_COMMENT} PROMPT_CONTENT="${PROMPT_CONTENT//\{\{PR_URL\}\}/${PR_URL}}" PROMPT_CONTENT="${PROMPT_CONTENT//\{\{PR_AUTHOR\}\}/${PR_AUTHOR}}" PROMPT_CONTENT="${PROMPT_CONTENT//\{\{PR_LABELS\}\}/${PR_LABELS:-none}}" - + # Write to temp file echo "$PROMPT_CONTENT" > "$output_file" - + # Verify prompt is not empty if [ ! -s "$output_file" ]; then echo "::error::Prompt file is empty after placeholder replacement" @@ -176,70 +223,78 @@ ${OVERRIDE_COMMENT} echo " 3. Placeholder replacement failed" return 1 fi - + echo "$output_file" } -# Run Gemini validation with retry logic -run_gemini() { +# Run ForgeCode validation with retry logic +run_forgecode() { local prompt_file="$1" local retry_count=0 local max_retries="${MAX_RETRIES:-2}" - + while [ $retry_count -le $max_retries ]; do - echo "Attempt $((retry_count + 1))/$((max_retries + 1)): Running Gemini validation..." - + echo "Attempt $((retry_count + 1))/$((max_retries + 1)): Running ForgeCode validation..." + # Debug: Check prompt file if [ ! -f "$prompt_file" ]; then echo "::error::Prompt file does not exist: $prompt_file" return 1 fi - + echo "Prompt file: $prompt_file" echo "Prompt file size: $(wc -c < "$prompt_file") bytes" echo "Prompt file lines: $(wc -l < "$prompt_file") lines" - + # Show detailed execution info - echo "๐Ÿค– Running Gemini CLI" + echo "๐Ÿค– Running ForgeCode CLI" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" - echo "Model: $GEMINI_MODEL" + echo "Provider: ${PROVIDER:-auto}" + echo "Model: ${MODEL}" echo "Prompt size: $(wc -c < "$prompt_file") bytes" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" echo "" - - # Run Gemini CLI and capture output + + # Configure ForgeCode with the specified model + if [ -n "${MODEL:-}" ]; then + echo "Configuring ForgeCode model: ${MODEL}" + forge config set --model "${MODEL}" 2>&1 || echo "Warning: Could not set model config" + fi + + # Run ForgeCode CLI and capture output set +e # Temporarily disable exit on error to capture exit code - - # Just use tee to show and save output - simple and effective - gemini -y -m "$GEMINI_MODEL" --prompt "$(cat "$prompt_file")" 2>&1 | tee /tmp/validation-full-output.md + + # Use -p flag for non-interactive mode with prompt from file + # Note: Removed --verbose to get clean output without debugging info + forge -p "$(cat "$prompt_file")" 2>&1 | tee /tmp/validation-full-output.md local exit_code=$? - + set -e # Re-enable exit on error - + echo "" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" - + if [ $exit_code -eq 0 ]; then - echo "โœ… Gemini CLI completed successfully (exit code: 0)" + echo "โœ… ForgeCode CLI completed successfully (exit code: 0)" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" return 0 else - echo "โŒ Gemini CLI failed (exit code: $exit_code)" + echo "โŒ ForgeCode CLI failed (exit code: $exit_code)" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" echo "" - + # Error details are shown above in the output echo "โš ๏ธ Check the output above for error details" - + # Check if it's a retryable error local is_retryable=false if [ -f /tmp/validation-full-output.md ] && grep -q -E "(429|503|timeout|rate limit)" /tmp/validation-full-output.md; then is_retryable=true fi - + if [ "$is_retryable" = true ]; then retry_count=$((retry_count + 1)) - + if [ $retry_count -le $max_retries ]; then wait_time=$((2 ** retry_count)) echo "โณ Rate limit or timeout detected. Retrying after ${wait_time} seconds..." @@ -255,50 +310,59 @@ run_gemini() { fi fi done - + return 1 } -# Extract validation report from Gemini output +# Extract validation report from ForgeCode output extract_report() { local full_output="$1" local report_file="/tmp/validation-report.md" + local clean_output="/tmp/validation-clean-output.md" + + # First, strip ANSI escape codes from the output + # This handles color codes and formatting that might interfere with pattern matching + if command -v sed &> /dev/null; then + sed 's/\x1b\[[0-9;]*m//g' "$full_output" > "$clean_output" + else + cp "$full_output" "$clean_output" + fi # Strategy 1: Look for "# PR Validation Report" header - if grep -q "# PR Validation Report" "$full_output"; then + if grep -q "# PR Validation Report" "$clean_output"; then echo "Extracting report using Strategy 1: PR Validation Report header" - sed -n '/# PR Validation Report/,$p' "$full_output" > "$report_file" + sed -n '/# PR Validation Report/,$p' "$clean_output" > "$report_file" return 0 fi # Strategy 2: Look for "## Summary" section - if grep -q "## Summary" "$full_output"; then + if grep -q "## Summary" "$clean_output"; then echo "Extracting report using Strategy 2: Summary section" - sed -n '/## Summary/,$p' "$full_output" > "$report_file" + sed -n '/## Summary/,$p' "$clean_output" > "$report_file" echo "# PR Validation Report" | cat - "$report_file" > /tmp/temp && mv /tmp/temp "$report_file" return 0 fi # Strategy 3: Look for "## Critical Violations" section - if grep -q "## Critical Violations" "$full_output"; then + if grep -q "## Critical Violations" "$clean_output"; then echo "Extracting report using Strategy 3: Critical Violations section" - sed -n '/## Critical Violations/,$p' "$full_output" > "$report_file" + sed -n '/## Critical Violations/,$p' "$clean_output" > "$report_file" echo "# PR Validation Report" | cat - "$report_file" > /tmp/temp && mv /tmp/temp "$report_file" return 0 fi # Strategy 4: Use full output with warning echo "::warning::Could not find report markers. Using full output." - echo "::group::Gemini Full Output (first 50 lines)" - head -50 "$full_output" || echo "Could not read output file" + echo "::group::ForgeCode Full Output (first 50 lines)" + head -50 "$clean_output" || echo "Could not read output file" echo "::endgroup::" - if [ ! -f "$full_output" ]; then - echo "::error::Full output file does not exist: $full_output" + if [ ! -f "$clean_output" ]; then + echo "::error::Clean output file does not exist: $clean_output" return 1 fi - cp "$full_output" "$report_file" + cp "$clean_output" "$report_file" if [ ! -f "$report_file" ]; then echo "::error::Failed to create report file: $report_file" @@ -309,16 +373,20 @@ extract_report() { return 0 } -# Parse validation results and set GitHub outputs +# Parse validation results from JSON and set GitHub outputs parse_results() { - local report_file="$1" + local json_file="/tmp/validation-report.json" - # Check for PASS/FAIL status + # Extract values from JSON local validation_status local validation_passed local critical_issues + local status + + # Get status from JSON + status=$(jq -r '.validationOutcome.status' "$json_file") - if grep -q -i "Status.*PASS" "$report_file" || grep -q "โœ…" "$report_file"; then + if [ "$status" = "PASS" ]; then validation_status="passed" validation_passed="true" else @@ -326,10 +394,8 @@ parse_results() { validation_passed="false" fi - # Count critical issues (looking for unchecked critical violations) - # Strip any whitespace/newlines and ensure we get a clean integer - critical_issues=$(grep -c "^- \[ \] \*\*" "$report_file" 2>/dev/null || echo "0") - critical_issues=$(echo "$critical_issues" | tr -d '\n\r' | tr -d ' ') + # Get critical issues count from JSON + critical_issues=$(jq -r '.validationOutcome.criticalIssuesCount' "$json_file") # Ensure we have a valid integer (default to 0 if empty or invalid) if ! [[ "$critical_issues" =~ ^[0-9]+$ ]]; then @@ -390,9 +456,9 @@ main() { prompt_with_replacements=$(prepare_prompt "$actual_prompt_file") echo "Prompt prepared: $prompt_with_replacements" - - # Run Gemini validation - if ! run_gemini "$prompt_with_replacements"; then + + # Run ForgeCode validation + if ! run_forgecode "$prompt_with_replacements"; then echo "::error::Validation execution failed" # Set failed outputs using heredoc delimiter @@ -412,30 +478,54 @@ main() { exit 1 fi - # Extract report from output - echo "::group::Extracting validation report" - echo "Full output file: /tmp/validation-full-output.md" - if [ -f "/tmp/validation-full-output.md" ]; then - echo "โœ… Full output file exists ($(wc -l < /tmp/validation-full-output.md) lines)" - else - echo "::error::Full output file does not exist!" + # Check if JSON report was created + echo "::group::Processing validation report" + + if [ ! -f "/tmp/validation-report.json" ]; then + echo "::error::JSON report file not created by ForgeCode" + echo "Expected file: /tmp/validation-report.json" + echo "" + echo "::group::ForgeCode output (for debugging)" + if [ -f "/tmp/validation-full-output.md" ]; then + head -100 "/tmp/validation-full-output.md" || echo "Could not read output file" + fi + echo "::endgroup::" + echo "::endgroup::" exit 1 fi - if ! extract_report "/tmp/validation-full-output.md"; then - echo "::error::Failed to extract validation report" + echo "โœ… JSON report found" + + # Validate JSON + if ! jq empty "/tmp/validation-report.json" 2>/dev/null; then + echo "::error::Invalid JSON in report file" + echo "::group::JSON content" + cat "/tmp/validation-report.json" || echo "Could not read JSON file" + echo "::endgroup::" echo "::endgroup::" exit 1 fi + + echo "โœ… JSON is valid" + + # Construct markdown report from JSON + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + if ! "$SCRIPT_DIR/construct-markdown.sh" "/tmp/validation-report.json" "/tmp/validation-report.md"; then + echo "::error::Failed to construct markdown from JSON" + echo "::endgroup::" + exit 1 + fi + + echo "โœ… Markdown report constructed" echo "::endgroup::" # Parse results and set outputs echo "Parsing validation results..." # Set GitHub outputs and get results - if [ -f "/tmp/validation-report.md" ]; then + if [ -f "/tmp/validation-report.json" ]; then # parse_results writes to GITHUB_OUTPUT and returns display values - results=$(parse_results "/tmp/validation-report.md") + results=$(parse_results) # Extract values for display (pipe-separated format) IFS='|' read -r validation_status validation_passed critical_issues <<< "$results" diff --git a/system-prompt.md b/system-prompt.md index 42e7cfc..ada1fc2 100644 --- a/system-prompt.md +++ b/system-prompt.md @@ -29,47 +29,65 @@ ## Output Format Requirements -### CRITICAL: Start Your Output - -**START YOUR OUTPUT DIRECTLY WITH:** `# PR Validation Report` - -**DO NOT** include in your output: - -- Your thinking process -- Standards content you fetched from Usable -- Git command outputs -- Tool execution logs -- Debug information -- Preamble or explanation of what you're about to do - -### Report Structure - -```markdown -# PR Validation Report - -## Summary -[Brief 2-3 sentence overview of the PR and overall assessment] - -## Critical Violations โŒ -[Must-fix issues that will fail the build] - -- **File**: `path/to/file.ts:42` -- **Issue**: [Clear description] -- **Why**: [Explanation of the violation] -- **Fix**: [Specific recommendation] - -## Important Issues โš ๏ธ -[Should-fix issues that won't fail the build] +### CRITICAL: Write JSON File + +**Write a JSON file to `/tmp/validation-report.json` following this schema:** + +```json +{ + "summary": "string (required) - Brief 2-3 sentence overview", + "criticalViolations": [ + { + "file": "string (required) - path/to/file.ts:42", + "issue": "string (required) - Clear description", + "why": "string (required) - Why this violates standards", + "fix": "string (required) - How to fix it", + "code": "string (optional) - Problematic code snippet" + } + ], + "importantIssues": [ + { + "file": "string (required)", + "issue": "string (required)", + "why": "string (required)", + "fix": "string (required)", + "code": "string (optional)" + } + ], + "suggestions": [ + { + "title": "string (required) - Brief title", + "description": "string (required) - Detailed explanation", + "file": "string (optional) - Related file path" + } + ], + "validationOutcome": { + "status": "string (required) - PASS or FAIL", + "criticalIssuesCount": "number (required)", + "importantIssuesCount": "number (required)", + "suggestionsCount": "number (required)", + "rationale": "string (optional) - Brief explanation" + }, + "overrideApplied": { + "deviation": "string (optional) - What was approved", + "justification": "string (optional) - Why it was needed", + "fragmentId": "string (optional) - Created fragment ID", + "fragmentTitle": "string (optional) - Fragment title", + "approvedBy": "string (optional) - Username" + }, + "metadata": { + "triggeredBy": "string (optional)", + "standardsChecked": ["array of strings (optional)"] + } +} +``` -## Suggestions โ„น๏ธ -[Nice-to-have improvements] +**Key Requirements**: -## Validation Outcome -- **Status**: PASS โœ… | FAIL โŒ -- **Critical Issues**: [count] -- **Important Issues**: [count] -- **Suggestions**: [count] -``` +- Arrays can be empty: `[]` +- Status must be exactly "PASS" or "FAIL" +- Counts must match array lengths +- Include line numbers in file paths when applicable ## Handling Override Comments @@ -91,7 +109,7 @@ Understand what the user is asking for: **Use `create-memory-fragment` with these parameters:** - **workspaceId**: `60c10ca2-4115-4c1a-b6d7-04ac39fd3938` (Flowcore workspace) -- **title**: `Approved Deviation: [Brief description]` (be specific and clear) +- **title**: `Approved Deviation: {Brief description}` (be specific and clear) - **fragmentTypeId**: `b06897e0-c39e-486b-8a9b-aab0ea260694` (solution type) - **repository**: `usable-pr-validator` (or the appropriate repo name from PR context) - **tags**: Always include `["repo:", "deviation", "approved"]` plus any relevant tech tags @@ -102,40 +120,27 @@ Understand what the user is asking for: # Approved Deviation ## Standard Deviated From -[Clear description of what standard/rule is being deviated from] +{Clear description of what standard/rule is being deviated from} ## Reason for Deviation -[Business or technical justification provided by the user] +{Business or technical justification provided by the user} ## Conditions and Limitations -[Any constraints or conditions for this deviation] +{Any constraints or conditions for this deviation} ## Approval Details -- **PR**: #[PR_NUMBER] - [PR_URL] -- **Approved by**: @[COMMENT_AUTHOR] -- **Date**: [Current date in YYYY-MM-DD format] -- **Repository**: [repo name] +- **PR**: #{PR_NUMBER} - {PR_URL} +- **Approved by**: @{COMMENT_AUTHOR} +- **Date**: {Current date in YYYY-MM-DD format} +- **Repository**: {repo name} ## Related Context -[Any additional context from the PR or comment] +{Any additional context from the PR or comment} ``` -### 3. Include Fragment Link in Report - -After creating the fragment, **include it in your validation report**: - -```markdown -## Override Applied - -A deviation from standards has been approved and documented: - -- **Deviation**: [Brief description] -- **Justification**: [User's reason] -- **Documentation**: Fragment created - [Fragment title] (ID: [fragment-id]) -- **Approved by**: @[username] +### 3. Include Override Information in JSON -This deviation has been recorded in the knowledge base for future reference. -``` +After creating the fragment, include the `overrideApplied` object in your JSON file with the fragment details. The markdown report will automatically generate an "Override Applied" section. ### 4. Adjust Validation Accordingly @@ -193,9 +198,12 @@ After documenting the deviation: - Identify the type of change (feature, fix, refactor, etc.) 2. **Analyze Changes** - - Get the diff: `git --no-pager diff origin/{base_branch}...{head_branch}` - - Identify all changed files - - Understand the scope and impact + - Review the changed files summary below: + + {{FILE_STATS}} + + - Use git commands to examine specific files: `git --no-pager diff origin/{base_branch}...{head_branch} -- ` + - Understand the scope and impact of changes **โš ๏ธ HANDLING GIT DIFF ERRORS:** diff --git a/test-local.sh b/test-local.sh new file mode 100755 index 0000000..ca3fea2 --- /dev/null +++ b/test-local.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Local testing script for PR validation +# This allows you to test the validator locally before pushing to GitHub + +# Parse command line arguments +TEST_MODE="full" +if [ "${1:-}" = "--mcp-only" ]; then + TEST_MODE="mcp" + echo "๐Ÿ”— Usable PR Validator - MCP Connection Test Only" + echo "==================================================" +else + echo "๐Ÿงช Usable PR Validator - Local Test" + echo "====================================" +fi +echo "" + +# Check required environment variables +MISSING_VARS=() + +if [ -z "${OPENROUTER_API_KEY:-}${ANTHROPIC_API_KEY:-}${OPENAI_API_KEY:-}" ]; then + MISSING_VARS+=("At least one of: OPENROUTER_API_KEY, ANTHROPIC_API_KEY, OPENAI_API_KEY") +fi + +if [ -z "${USABLE_API_TOKEN:-}" ]; then + MISSING_VARS+=("USABLE_API_TOKEN") +fi + +if [ ${#MISSING_VARS[@]} -gt 0 ]; then + echo "โŒ Missing required environment variables:" + for var in "${MISSING_VARS[@]}"; do + echo " - $var" + done + echo "" + echo "Please set these in your environment or .env file" + echo "" + echo "Example:" + echo " export OPENROUTER_API_KEY='your-key-here'" + echo " export USABLE_API_TOKEN='your-token-here'" + exit 1 +fi + +# Set default values for local testing +export PROVIDER="${PROVIDER:-auto}" +export MODEL="${MODEL:-anthropic/claude-haiku-4.5}" +export WORKSPACE_ID="${WORKSPACE_ID:-60c10ca2-4115-4c1a-b6d7-04ac39fd3938}" # Flowcore workspace +export USABLE_URL="${USABLE_URL:-https://usable.dev}" + +# Git configuration (you can override these) +export BASE_BRANCH="${BASE_BRANCH:-main}" +export HEAD_BRANCH="${HEAD_BRANCH:-$(git rev-parse --abbrev-ref HEAD)}" + +# PR metadata (simulated for local testing) +export PR_NUMBER="${PR_NUMBER:-0}" +export PR_TITLE="${PR_TITLE:-Local Test: $(git log -1 --pretty=%s)}" +export PR_DESCRIPTION="${PR_DESCRIPTION:-Testing PR validation locally}" +export PR_URL="${PR_URL:-https://github.com/local/test}" +export PR_AUTHOR="${PR_AUTHOR:-$(git config user.name)}" +export PR_LABELS="${PR_LABELS:-test,local}" + +# Other settings +export USE_DYNAMIC_PROMPTS="${USE_DYNAMIC_PROMPTS:-false}" +# Don't set CUSTOM_PROMPT_FILE by default - let fetch-prompt.sh use only hardcoded + MCP prompts +# Users can set this if they want to test with a custom prompt file +export CUSTOM_PROMPT_FILE="${CUSTOM_PROMPT_FILE:-}" +export PROMPT_FILE="${PROMPT_FILE:-./system-prompt.md}" +export MAX_RETRIES="${MAX_RETRIES:-2}" +export ALLOW_WEB_FETCH="${ALLOW_WEB_FETCH:-false}" +# Don't export these if they're empty - ForgeCode treats empty strings as file paths +if [ -n "${OVERRIDE_COMMENT:-}" ]; then + export OVERRIDE_COMMENT +fi +if [ -n "${COMMENT_AUTHOR:-}" ]; then + export COMMENT_AUTHOR +fi + +# GitHub Actions environment variables (simulated) +export GITHUB_OUTPUT="${GITHUB_OUTPUT:-/tmp/github-output.txt}" +export GITHUB_ENV="${GITHUB_ENV:-/tmp/github-env.txt}" +touch "$GITHUB_OUTPUT" "$GITHUB_ENV" + +echo "๐Ÿ“‹ Test Configuration" +echo "====================" +echo "Provider: $PROVIDER" +echo "Model: $MODEL" +echo "Base Branch: $BASE_BRANCH" +echo "Head Branch: $HEAD_BRANCH" +echo "Workspace: $WORKSPACE_ID" +echo "" + +# Check if ForgeCode is installed +if ! command -v forge &> /dev/null; then + echo "โš ๏ธ ForgeCode CLI not found. Installing..." + npm install -g forgecode@latest + echo "โœ… ForgeCode installed" + echo "" +fi + +# Verify git refs are available +echo "๐Ÿ” Verifying Git Setup" +echo "=====================" +if ! git rev-parse "origin/$BASE_BRANCH" >/dev/null 2>&1; then + echo "โš ๏ธ Base branch origin/$BASE_BRANCH not found. Fetching..." + git fetch origin "$BASE_BRANCH" +fi + +if ! git rev-parse "$HEAD_BRANCH" >/dev/null 2>&1; then + echo "โŒ Head branch $HEAD_BRANCH not found" + echo "Current branch: $(git rev-parse --abbrev-ref HEAD)" + exit 1 +fi + +echo "โœ… Git refs verified" +echo "" + +# Run provider setup +echo "๐Ÿ” Setting up Provider" +echo "=====================" +./scripts/setup-provider.sh +echo "" + +# Run MCP setup +echo "๐Ÿ”— Setting up MCP" +echo "================" +# For local testing, USABLE_URL and USABLE_API_TOKEN are in the environment +./scripts/setup-mcp.sh +echo "" + +# If MCP-only test mode, run the MCP connection test and exit +if [ "$TEST_MODE" = "mcp" ]; then + echo "๐Ÿงช Running MCP Connection Test" + echo "==============================" + ./scripts/test-mcp-connection.sh + exit $? +fi + +# Check if we should fetch dynamic prompts or just prepare prompts +echo "๐Ÿ“ฅ Fetching/Preparing Prompts" +echo "=============================" +# Set ACTION_PATH for local testing (points to repo root where system-prompt.md is) +export ACTION_PATH="$(pwd)" +./scripts/fetch-prompt.sh +echo "" + +# Run validation +echo "๐Ÿš€ Running Validation" +echo "====================" +echo "" +./scripts/validate.sh + +# Show results +echo "" +echo "๐Ÿ“Š Validation Results" +echo "====================" +if [ -f "$GITHUB_OUTPUT" ]; then + echo "GitHub Outputs:" + cat "$GITHUB_OUTPUT" + echo "" +fi + +if [ -f /tmp/validation-report.md ]; then + echo "Validation Report:" + echo "==================" + cat /tmp/validation-report.md + echo "" +fi + +echo "โœ… Local test completed" +echo "" +echo "Artifacts:" +echo " - Full output: /tmp/validation-full-output.md" +echo " - Report: /tmp/validation-report.md" +echo " - GitHub outputs: $GITHUB_OUTPUT"