From 2f4e59e51817eeabac5fd4c953f44809fb16a5e6 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Wed, 22 Oct 2025 22:39:06 +0100 Subject: [PATCH 01/41] refactor(action.yml, README.md, workflows, scripts): transition from Gemini to ForgeCode integration - Updated action.yml to replace Gemini model references with ForgeCode and added support for multiple LLM providers (OpenRouter, OpenAI, Anthropic). - Modified README.md to reflect the new AI provider options and updated features. - Introduced new scripts for provider setup and validation, ensuring compatibility with the ForgeCode CLI. - Enhanced workflows to utilize the new provider and model inputs, streamlining the validation process. - Added local testing script for easier validation before pushing changes to GitHub Actions. --- .github/workflows/comment-revalidation.yml | 30 +++- .github/workflows/test-reusable-workflow.yml | 19 ++- .github/workflows/test.yml | 22 +-- README.md | 134 +++++++++++----- action.yml | 46 +++--- forge.yaml | 2 + scripts/setup-mcp.sh | 53 ++++--- scripts/setup-provider.sh | 76 +++++++++ scripts/validate.sh | 126 ++++++++++----- system-prompt.md | 55 +++---- test-local.sh | 155 +++++++++++++++++++ 11 files changed, 552 insertions(+), 166 deletions(-) create mode 100644 forge.yaml create mode 100755 scripts/setup-provider.sh create mode 100755 test-local.sh diff --git a/.github/workflows/comment-revalidation.yml b/.github/workflows/comment-revalidation.yml index 69869f5..a31a23b 100644 --- a/.github/workflows/comment-revalidation.yml +++ b/.github/workflows/comment-revalidation.yml @@ -25,11 +25,16 @@ on: required: false type: string default: '' - gemini-model: - description: 'Gemini model to use' + provider: + description: 'LLM provider (openrouter, anthropic, openai, auto)' required: false type: string - default: 'gemini-2.5-flash' + default: 'auto' + model: + description: 'Model to use' + required: false + type: string + default: 'openai/gpt-oss-120b' comment-title: description: 'Title for the validation comment' required: false @@ -41,9 +46,15 @@ on: type: boolean default: false secrets: - GEMINI_SERVICE_ACCOUNT_KEY: - description: 'Base64-encoded Gemini service account key' - required: true + OPENROUTER_API_KEY: + description: 'OpenRouter API key (or ANTHROPIC_API_KEY/OPENAI_API_KEY)' + required: false + ANTHROPIC_API_KEY: + description: 'Anthropic API key' + required: false + OPENAI_API_KEY: + description: 'OpenAI API key' + required: false USABLE_API_TOKEN: description: 'Usable API token' required: true @@ -273,13 +284,16 @@ jobs: prompt-fragment-id: ${{ inputs.prompt-fragment-id || '' }} workspace-id: ${{ inputs.workspace-id || '60c10ca2-4115-4c1a-b6d7-04ac39fd3938' }} base-ref: ${{ steps.base-ref.outputs.base-ref }} - gemini-model: ${{ inputs.gemini-model || 'gemini-2.5-flash' }} + provider: ${{ inputs.provider || 'auto' }} + model: ${{ inputs.model || 'openai/gpt-oss-120b' }} comment-title: ${{ inputs.comment-title || '๐Ÿ”„ Revalidation (Comment Triggered)' }} comment-mode: 'create' fail-on-critical: ${{ inputs.fail-on-critical || false }} override-comment: ${{ needs.check-trigger.outputs.comment-body }} env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} COMMENT_AUTHOR: ${{ github.event.comment.user.login }} diff --git a/.github/workflows/test-reusable-workflow.yml b/.github/workflows/test-reusable-workflow.yml index 3198d74..5447aac 100644 --- a/.github/workflows/test-reusable-workflow.yml +++ b/.github/workflows/test-reusable-workflow.yml @@ -23,11 +23,12 @@ jobs: with: workspace-id: '60c10ca2-4115-4c1a-b6d7-04ac39fd3938' prompt-file: 'templates/basic-validation.md' - gemini-model: 'gemini-2.5-flash' + provider: 'auto' + model: 'openai/gpt-oss-120b' comment-title: '๐Ÿงช Reusable Workflow Test' fail-on-critical: false secrets: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} permissions: contents: read @@ -62,22 +63,26 @@ jobs: **Test Configuration**: - Workspace ID: 60c10ca2-4115-4c1a-b6d7-04ac39fd3938 - Prompt File: templates/basic-validation.md - - Model: gemini-2.5-flash + - Provider: auto + - Model: openai/gpt-oss-120b - Comment Title: ๐Ÿงช Reusable Workflow Test - + **Result**: ${{ needs.test-reusable.result }} - + **Next Steps**: - If successful: Reusable workflow is ready for users - If failed: Check workflow logs for errors - + **How Users Will Call It**: \`\`\`yaml uses: flowcore/usable-pr-validator/.github/workflows/comment-revalidation.yml@v1 with: workspace-id: 'their-workspace-uuid' + # Optional - defaults to auto and openai/gpt-oss-120b + # provider: 'openrouter' # or 'anthropic', 'openai', 'auto' + # model: 'anthropic/claude-3.7-sonnet' # for higher quality secrets: - GEMINI_SERVICE_ACCOUNT_KEY: \${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: \${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: \${{ secrets.USABLE_API_TOKEN }} \`\`\` EOF diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d821987..e345718 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -182,16 +182,16 @@ jobs: - name: Verify Cleanup Steps run: | # Check that action.yml has cleanup steps - if ! grep -q "rm -f /tmp/service-account.json" action.yml; then - echo "::error::Missing cleanup for service-account.json" + if ! grep -q "rm -f /tmp/forge-config.yaml" action.yml; then + echo "::error::Missing cleanup for forge-config.yaml" exit 1 fi - - if ! grep -q "rm -f /tmp/gemini-settings.json" action.yml; then - echo "::error::Missing cleanup for gemini-settings.json" + + if ! grep -q "rm -f /tmp/validation-.*\.md" action.yml; then + echo "::error::Missing cleanup for validation files" exit 1 fi - + echo "โœ… Cleanup steps verified" integration-test: @@ -313,7 +313,7 @@ jobs: comment-mode: 'update' fail-on-critical: false env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} - name: Debug Test Results @@ -321,7 +321,7 @@ jobs: run: | echo "Test result: ${{ steps.test-action.outcome }}" if [ -f "/tmp/validation-full-output.md" ]; then - echo "::group::Full Gemini Output" + echo "::group::Full ForgeCode Output" cat /tmp/validation-full-output.md echo "::endgroup::" else @@ -430,7 +430,7 @@ jobs: comment-mode: 'update' fail-on-critical: false env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} - name: Debug Documentation Results @@ -438,7 +438,7 @@ jobs: run: | echo "Docs test result: ${{ steps.docs-action.outcome }}" if [ -f "/tmp/validation-full-output.md" ]; then - echo "::group::Full Gemini Output" + echo "::group::Full ForgeCode Output" cat /tmp/validation-full-output.md echo "::endgroup::" else @@ -569,7 +569,7 @@ jobs: comment-mode: 'update' fail-on-critical: false env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} COMMENT_AUTHOR: 'test-user' diff --git a/README.md b/README.md index 074b038..37cbdd4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ๐Ÿค– Usable PR Validator -> Validate Pull Requests against your Usable knowledge base standards using Google Gemini AI +> Validate Pull Requests against your Usable knowledge base standards using AI (OpenRouter, Anthropic, OpenAI, and more) [![GitHub Marketplace](https://img.shields.io/badge/Marketplace-Usable%20PR%20Validator-blue?logo=github)](https://github.com/marketplace/actions/usable-pr-validator) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) @@ -8,9 +8,10 @@ ## โœจ Features -- ๐Ÿง  **AI-Powered Validation**: Uses Google Gemini to understand context and architectural patterns +- ๐Ÿง  **AI-Powered Validation**: Uses advanced AI models (Claude, GPT-4, Llama, etc.) via ForgeCode +- ๐Ÿ”Œ **Multi-Provider Support**: OpenRouter, Anthropic, OpenAI, and more - bring your own API key - ๐Ÿ“š **Usable Integration**: Validate PRs against your team's knowledge base stored in Usable -- ๐Ÿ”Œ **MCP Protocol**: Connects directly to Usable's MCP server for real-time standards +- ๐Ÿ”— **MCP Protocol**: Connects directly to Usable's MCP server for real-time standards - ๐ŸŽฏ **System Prompts**: Organization-wide validation standards fetched from Usable and auto-merged - ๐Ÿš€ **Dynamic Prompts**: Fetch latest validation prompts from Usable API (no static files needed!) - ๐Ÿ’ฌ **Comment-Triggered Revalidation**: Mention `@usable` in PR comments to trigger revalidation with context @@ -25,10 +26,12 @@ ### Prerequisites -1. A Google Cloud project with Vertex AI API enabled -2. A service account key with Vertex AI permissions -3. A Usable account with API token ([get one at usable.dev](https://usable.dev)) -4. GitHub repository with pull requests +1. An API key from one of: + - [OpenRouter](https://openrouter.ai) (recommended - access to 200+ models) + - [Anthropic](https://console.anthropic.com) (Claude models) + - [OpenAI](https://platform.openai.com) (GPT models) +2. A Usable account with API token ([get one at usable.dev](https://usable.dev)) +3. GitHub repository with pull requests ### Step 1: Create Validation Prompt @@ -57,12 +60,7 @@ git diff origin/{{BASE_BRANCH}}...{{HEAD_BRANCH}} Go to your repository Settings โ†’ Secrets โ†’ Actions and add: -- `GEMINI_SERVICE_ACCOUNT_KEY`: Base64-encoded service account JSON key - - ```bash - cat service-account.json | base64 - ``` - +- `OPENROUTER_API_KEY`: Your OpenRouter API key (or `ANTHROPIC_API_KEY` / `OPENAI_API_KEY`) - `USABLE_API_TOKEN`: Your Usable API token (get from [usable.dev](https://usable.dev) โ†’ Settings โ†’ API Tokens) ### Step 3: Create Workflow @@ -87,18 +85,70 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - + - uses: flowcore/usable-pr-validator@latest with: prompt-file: '.github/prompts/pr-validation.md' workspace-id: 'your-workspace-uuid' + # Optional: specify provider and model (defaults to auto-detect and openai/gpt-oss-120b) + # provider: 'openrouter' # or 'anthropic', 'openai', 'auto' + # model: 'anthropic/claude-3.7-sonnet' # for higher quality env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} ``` That's it! Your PRs will now be validated automatically. ๐ŸŽ‰ +## ๐Ÿงช Local Testing + +You can test the PR validator locally before pushing to GitHub Actions: + +### Required Environment Variables + +```bash +# At least one of these API keys (depending on your provider) +export OPENROUTER_API_KEY='your-openrouter-key' # Recommended +# OR +export ANTHROPIC_API_KEY='your-anthropic-key' # For direct Anthropic +# OR +export OPENAI_API_KEY='your-openai-key' # For OpenAI + +# Required +export USABLE_API_TOKEN='your-usable-token' + +# Optional (can be set in the script or use defaults) +export PROVIDER='auto' # or 'openrouter', 'anthropic', 'openai' +export MODEL='openai/gpt-oss-120b' # default model (fast and cost-effective) +export WORKSPACE_ID='your-workspace-uuid' # Usable workspace +export BASE_BRANCH='main' # base branch for diff +export HEAD_BRANCH='feature-branch' # current branch +``` + +### Run Local Test + +```bash +# Make sure you're on the branch you want to test +git checkout your-feature-branch + +# Run the test +./test-local.sh +``` + +The script will: +1. Verify all required environment variables are set +2. Install ForgeCode CLI if not already installed +3. Fetch the necessary git refs +4. Set up the provider authentication +5. Run the validation +6. Display the results and save artifacts to `/tmp/` + +### Viewing Results + +- **Full output**: `/tmp/validation-full-output.md` +- **Validation report**: `/tmp/validation-report.md` +- **GitHub outputs**: `/tmp/github-output.txt` + ## ๐Ÿ“– Configuration ### Inputs @@ -110,8 +160,9 @@ That's it! Your PRs will now be validated automatically. ๐ŸŽ‰ | `prompt-fragment-id` | Usable fragment UUID to use as prompt (required when `use-dynamic-prompts` is `true`) | โœ“ (with dynamic prompts) | | | `workspace-id` | Usable workspace UUID (required - used to fetch MCP system prompt) | โœ“ | | | `merge-custom-prompt` | Merge fetched Usable prompt with custom `prompt-file` (only when both are provided) | | `true` | -| `gemini-model` | Gemini model to use (`gemini-2.5-flash`, `gemini-2.0-flash`, `gemini-2.5-pro`) | | `gemini-2.5-flash` | -| `service-account-key-secret` | Secret name for service account key | | `GEMINI_SERVICE_ACCOUNT_KEY` | +| `provider` | LLM provider: `openrouter`, `openai`, `anthropic`, or `auto` (auto-detect from env vars) | | `auto` | +| `model` | Model to use (e.g., `openai/gpt-oss-120b`, `anthropic/claude-3.7-sonnet`, `openai/gpt-4`, `meta-llama/llama-3.3-70b-instruct`) | | `openai/gpt-oss-120b` | +| `api-key-secret` | Name of secret containing API key | | `OPENROUTER_API_KEY` | | `mcp-server-url` | Usable MCP server URL | | `https://usable.dev/api/mcp` | | `mcp-token-secret` | Secret name for Usable API token | | `USABLE_API_TOKEN` | | `fail-on-critical` | Fail build on critical violations | | `true` | @@ -180,7 +231,7 @@ Instead of maintaining static prompt files, you can now fetch prompts dynamicall prompt-fragment-id: 'user-prompt-uuid' workspace-id: 'your-workspace-uuid' env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} # Static user prompt file (most common) @@ -189,7 +240,7 @@ Instead of maintaining static prompt files, you can now fetch prompts dynamicall prompt-file: '.github/prompts/pr-validation.md' workspace-id: 'your-workspace-uuid' env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} ``` @@ -212,22 +263,36 @@ Instead of maintaining static prompt files, you can now fetch prompts dynamicall prompt-file: '.github/prompts/validate.md' workspace-id: 'your-workspace-uuid' env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} ``` -### With Custom MCP Server +### With Anthropic Claude ```yaml - uses: flowcore/usable-pr-validator@latest with: prompt-file: '.github/prompts/validate.md' workspace-id: 'your-workspace-uuid' - mcp-server-url: 'https://your-custom-mcp.com/api/mcp' - mcp-token-secret: 'YOUR_CUSTOM_TOKEN' + provider: 'anthropic' + model: 'claude-3.7-sonnet' env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} - YOUR_CUSTOM_TOKEN: ${{ secrets.YOUR_MCP_TOKEN }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} +``` + +### With OpenAI GPT + +```yaml +- uses: flowcore/usable-pr-validator@latest + with: + prompt-file: '.github/prompts/validate.md' + workspace-id: 'your-workspace-uuid' + provider: 'openai' + model: 'gpt-4' + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} ``` ### Advanced Configuration @@ -237,17 +302,16 @@ Instead of maintaining static prompt files, you can now fetch prompts dynamicall with: prompt-file: '.github/validation/standards.md' workspace-id: 'your-workspace-uuid' - gemini-model: 'gemini-2.5-pro' - service-account-key-secret: 'MY_GEMINI_KEY' - mcp-server-url: 'https://confluence.company.com/api/mcp' - mcp-token-secret: 'CONFLUENCE_TOKEN' + provider: 'openrouter' + model: 'anthropic/claude-3.7-sonnet' + mcp-server-url: 'https://usable.dev/api/mcp' fail-on-critical: true comment-mode: 'update' artifact-retention-days: 90 max-retries: 3 env: - MY_GEMINI_KEY: ${{ secrets.GOOGLE_AI_KEY }} - CONFLUENCE_TOKEN: ${{ secrets.CONF_API_TOKEN }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} ``` ### Multiple Validation Stages @@ -269,23 +333,23 @@ jobs: workspace-id: 'your-workspace-uuid' comment-title: 'Backend Validation' # Creates unique comment env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} - + validate-frontend: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - + - uses: flowcore/usable-pr-validator@latest with: prompt-file: '.github/prompts/frontend-standards.md' workspace-id: 'your-workspace-uuid' comment-title: 'Frontend Validation' # Creates unique comment env: - GEMINI_SERVICE_ACCOUNT_KEY: ${{ secrets.GEMINI_SERVICE_ACCOUNT_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} ``` diff --git a/action.yml b/action.yml index 571f5e1..000d04d 100644 --- a/action.yml +++ b/action.yml @@ -1,5 +1,5 @@ name: 'Usable PR Validator' -description: 'Validate Pull Requests against your Usable knowledge base standards using Google Gemini AI and Usable MCP integration' +description: 'Validate Pull Requests against your Usable knowledge base standards using ForgeCode with OpenRouter, OpenAI, Anthropic, or other LLM providers' author: 'Flowcore' branding: @@ -27,14 +27,18 @@ inputs: description: 'User comment requesting validation override or clarification (typically from @usable mention)' required: false default: '' - gemini-model: - description: 'Gemini model to use (gemini-2.5-flash, gemini-2.0-flash, gemini-2.5-pro)' + provider: + description: 'LLM provider: openrouter, openai, anthropic, or auto (auto-detect from env vars)' required: false - default: 'gemini-2.5-flash' - service-account-key-secret: - description: 'Name of secret containing base64-encoded Gemini service account key' + default: 'auto' + model: + description: 'Model to use (e.g., openai/gpt-oss-120b, anthropic/claude-3.7-sonnet, openai/gpt-4, meta-llama/llama-3.3-70b-instruct)' required: false - default: 'GEMINI_SERVICE_ACCOUNT_KEY' + default: 'openai/gpt-oss-120b' + api-key-secret: + description: 'Name of secret containing API key (OPENROUTER_API_KEY, OPENAI_API_KEY, or ANTHROPIC_API_KEY)' + required: false + default: 'OPENROUTER_API_KEY' mcp-server-url: description: 'HTTP URL of MCP server' required: false @@ -164,12 +168,12 @@ runs: with: node-version: '20' - - name: Install Gemini CLI + - name: Install ForgeCode CLI shell: bash run: | - echo "::group::Installing Gemini CLI" - npm install -g @google/gemini-cli@0.7.0 - echo "โœ… Gemini CLI installed" + echo "::group::Installing ForgeCode CLI" + npm install -g forgecode@latest + echo "โœ… ForgeCode CLI installed" echo "::endgroup::" - name: Setup Git @@ -276,12 +280,13 @@ runs: echo "โœ… Git configured" echo "::endgroup::" - - name: Setup Gemini Authentication + - name: Setup LLM Provider Authentication shell: bash env: - SECRET_NAME: ${{ inputs.service-account-key-secret }} + PROVIDER: ${{ inputs.provider }} + API_KEY_SECRET: ${{ inputs.api-key-secret }} run: | - ${{ github.action_path }}/scripts/setup-gemini.sh + ${{ github.action_path }}/scripts/setup-provider.sh - name: Setup MCP Server shell: bash @@ -311,7 +316,8 @@ runs: env: PROMPT_FILE: ${{ inputs.prompt-file }} USE_DYNAMIC_PROMPTS: ${{ inputs.use-dynamic-prompts }} - GEMINI_MODEL: ${{ inputs.gemini-model }} + PROVIDER: ${{ inputs.provider }} + MODEL: ${{ inputs.model }} MAX_RETRIES: ${{ inputs.max-retries }} PR_NUMBER: ${{ github.event.pull_request.number }} PR_TITLE: ${{ github.event.pull_request.title }} @@ -349,9 +355,10 @@ runs: const markerId = commentTitle.toLowerCase().replace(/[^a-z0-9]+/g, '-'); const marker = ``; - const commentBody = marker + '\n## ๐Ÿค– ' + commentTitle + '\n\n' + + const commentBody = marker + '\n## ๐Ÿค– ' + commentTitle + '\n\n' + report + '\n\n---\n
\n๐Ÿ“Š Validation Statistics\n\n' + - '- **Model**: Google Gemini ${{ inputs.gemini-model }}\n' + + '- **Provider**: ${{ inputs.provider }}\n' + + '- **Model**: ${{ inputs.model }}\n' + '- **Standards Source**: ${{ inputs.mcp-server-url }}\n' + '- **Commit**: ' + context.payload.pull_request.head.sha.substring(0, 7) + '\n' + '- **Triggered by**: @' + context.actor + '\n\n' + @@ -408,9 +415,8 @@ runs: shell: bash run: | # Remove temporary files containing sensitive data - rm -f /tmp/service-account.json - rm -f /tmp/gemini-settings.json - rm -f /tmp/gemini-stderr.log + rm -f /tmp/forge-config.yaml + rm -f /tmp/validation-*.md echo "โœ… Temporary files cleaned up" - name: Fail on Critical Violations diff --git a/forge.yaml b/forge.yaml new file mode 100644 index 0000000..e0ec477 --- /dev/null +++ b/forge.yaml @@ -0,0 +1,2 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/antinomyhq/forge/refs/heads/main/forge.schema.json +{} diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index d2a6580..615a1c1 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -3,12 +3,21 @@ set -euo pipefail echo "::group::Setting up MCP Server Integration" -# Get the MCP token from environment using the secret name -MCP_TOKEN="${!MCP_SECRET_NAME:-}" +# Get the MCP token from environment +# In GitHub Actions: MCP_SECRET_NAME points to the env var name (e.g., USABLE_API_TOKEN) +# In local testing: Just use USABLE_API_TOKEN directly +if [ -n "${MCP_SECRET_NAME:-}" ]; then + # GitHub Actions mode: indirect reference + MCP_TOKEN="${!MCP_SECRET_NAME:-}" +else + # Local testing mode: direct token + MCP_TOKEN="${USABLE_API_TOKEN:-}" +fi if [ -z "$MCP_TOKEN" ]; then - echo "::error::MCP token not found in environment variable: $MCP_SECRET_NAME" - echo "Please ensure the secret is set in your workflow: env.$MCP_SECRET_NAME" + echo "::error::MCP token not found" + echo "For GitHub Actions: Ensure env.${MCP_SECRET_NAME:-USABLE_API_TOKEN} is set" + echo "For local testing: Export USABLE_API_TOKEN environment variable" exit 1 fi @@ -18,33 +27,31 @@ if [ -z "$MCP_URL" ]; then exit 1 fi -# Create Gemini settings file with MCP configuration -cat > /tmp/gemini-settings.json < /tmp/forge-mcp-config.yaml <> $GITHUB_ENV +# Export as environment variable for forge to pick up +export FORGE_MCP_CONFIG="/tmp/forge-mcp-config.yaml" +if [ -n "${GITHUB_ENV:-}" ]; then + echo "FORGE_MCP_CONFIG=/tmp/forge-mcp-config.yaml" >> "$GITHUB_ENV" +fi -echo "โœ… MCP server configured" +echo "โœ… MCP server configured for ForgeCode" echo " URL: $MCP_URL" -echo " Settings file: /tmp/gemini-settings.json" +echo " Config file: /tmp/forge-mcp-config.yaml" # Debug: Show settings file content (mask token) echo " Configuration preview:" -cat /tmp/gemini-settings.json | sed 's/"Bearer [^"]*"/"Bearer ***MASKED***"/g' | sed 's/^/ /' +cat /tmp/forge-mcp-config.yaml | sed 's/Bearer [^"]*$/Bearer ***MASKED***/g' | sed 's/^/ /' echo "::endgroup::" diff --git a/scripts/setup-provider.sh b/scripts/setup-provider.sh new file mode 100755 index 0000000..363ae93 --- /dev/null +++ b/scripts/setup-provider.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "::group::Setting up LLM Provider Authentication" + +# Determine which provider to use +PROVIDER="${PROVIDER:-auto}" + +echo "Provider mode: $PROVIDER" + +# Function to check if an API key is available +check_api_key() { + local key_name="$1" + local key_value="${!key_name:-}" + + if [ -n "$key_value" ]; then + echo "โœ… $key_name is available" + return 0 + else + echo "โŒ $key_name is not set" + return 1 + fi +} + +# Auto-detect provider if set to auto +if [ "$PROVIDER" = "auto" ]; then + echo "Auto-detecting provider from environment variables..." + + if check_api_key "OPENROUTER_API_KEY"; then + PROVIDER="openrouter" + echo "๐ŸŽฏ Auto-detected provider: OpenRouter" + elif check_api_key "ANTHROPIC_API_KEY"; then + PROVIDER="anthropic" + echo "๐ŸŽฏ Auto-detected provider: Anthropic" + elif check_api_key "OPENAI_API_KEY"; then + PROVIDER="openai" + echo "๐ŸŽฏ Auto-detected provider: OpenAI" + else + echo "::error::No API key found in environment" + echo "Please set one of: OPENROUTER_API_KEY, ANTHROPIC_API_KEY, OPENAI_API_KEY" + exit 1 + fi +fi + +# Validate that the required API key is present +case "$PROVIDER" in + openrouter) + if ! check_api_key "OPENROUTER_API_KEY"; then + echo "::error::OPENROUTER_API_KEY is required for OpenRouter provider" + exit 1 + fi + ;; + anthropic) + if ! check_api_key "ANTHROPIC_API_KEY"; then + echo "::error::ANTHROPIC_API_KEY is required for Anthropic provider" + exit 1 + fi + ;; + openai) + if ! check_api_key "OPENAI_API_KEY"; then + echo "::error::OPENAI_API_KEY is required for OpenAI provider" + exit 1 + fi + ;; + *) + echo "::error::Unknown provider: $PROVIDER" + echo "Supported providers: openrouter, anthropic, openai, auto" + exit 1 + ;; +esac + +# Export provider for downstream scripts +echo "PROVIDER=$PROVIDER" >> "$GITHUB_ENV" 2>/dev/null || export PROVIDER="$PROVIDER" + +echo "โœ… Provider authentication configured: $PROVIDER" +echo "::endgroup::" diff --git a/scripts/validate.sh b/scripts/validate.sh index 9a689e2..4db10e0 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -100,11 +100,53 @@ verify_git_refs() { fi } +# Get file stats for the PR +get_file_stats() { + local base="${BASE_BRANCH}" + local head="${HEAD_BRANCH}" + + echo "Getting file statistics for PR..." + + # Try to get the diff stat + local file_stats="" + if git diff --stat "origin/$base...origin/$head" 2>/dev/null; then + file_stats=$(git diff --stat "origin/$base...origin/$head" 2>/dev/null || echo "Unable to retrieve file statistics") + elif git diff --stat "$base...$head" 2>/dev/null; then + file_stats=$(git diff --stat "$base...$head" 2>/dev/null || echo "Unable to retrieve file statistics") + else + file_stats="Unable to retrieve file statistics. Use git commands to explore changes." + fi + + # Also get list of changed files + local changed_files="" + if git diff --name-only "origin/$base...origin/$head" 2>/dev/null; then + changed_files=$(git diff --name-only "origin/$base...origin/$head" 2>/dev/null | head -100) + elif git diff --name-only "$base...$head" 2>/dev/null; then + changed_files=$(git diff --name-only "$base...$head" 2>/dev/null | head -100) + fi + + # Format the output + local output="## Files Changed + +\`\`\` +$file_stats +\`\`\` + +## Changed Files List +\`\`\` +$changed_files +\`\`\` + +**Note**: Use \`git --no-pager diff origin/$base...origin/$head -- \` to examine specific files." + + echo "$output" +} + # Prepare prompt with placeholder replacement -# +# # Uses bash native string replacement for simplicity and safety. # For current scope (8 placeholders), this is efficient and readable. -# +# # Alternative approaches considered: # - envsubst: Would require careful escaping of $ symbols in prompts # - sed: More complex escaping, harder to maintain @@ -117,7 +159,7 @@ verify_git_refs() { prepare_prompt() { local prompt_file="$1" local output_file="/tmp/validation-prompt.txt" - + # Create PR context block PR_CONTEXT="**PR #${PR_NUMBER}**: ${PR_TITLE} @@ -129,7 +171,7 @@ prepare_prompt() { ${PR_DESCRIPTION:-No description provided}" # Add override comment if provided - if [ -n "$OVERRIDE_COMMENT" ]; then + if [ -n "${OVERRIDE_COMMENT:-}" ]; then PR_CONTEXT="${PR_CONTEXT} **๐Ÿ”„ Override/Clarification Comment** (from @${COMMENT_AUTHOR:-unknown}): @@ -147,13 +189,18 @@ ${OVERRIDE_COMMENT} web_fetch_policy="**Web fetch is DISABLED** for this validation. DO NOT use the \`web_fetch\` tool or attempt to download content from URLs. All validation must be performed using only the git repository contents, PR context, and Usable MCP knowledge base." fi + # Get file statistics + local file_stats + file_stats=$(get_file_stats) + # Read prompt template PROMPT_CONTENT=$(cat "$prompt_file") - + # Replace placeholders using bash string replacement (NOT sed) # This handles special characters safely PROMPT_CONTENT="${PROMPT_CONTENT//\{\{WEB_FETCH_POLICY\}\}/${web_fetch_policy}}" PROMPT_CONTENT="${PROMPT_CONTENT//\{\{PR_CONTEXT\}\}/${PR_CONTEXT}}" + PROMPT_CONTENT="${PROMPT_CONTENT//\{\{FILE_STATS\}\}/${file_stats}}" PROMPT_CONTENT="${PROMPT_CONTENT//\{\{BASE_BRANCH\}\}/${BASE_BRANCH}}" PROMPT_CONTENT="${PROMPT_CONTENT//\{\{HEAD_BRANCH\}\}/${HEAD_BRANCH}}" PROMPT_CONTENT="${PROMPT_CONTENT//\{\{PR_TITLE\}\}/${PR_TITLE}}" @@ -162,10 +209,10 @@ ${OVERRIDE_COMMENT} PROMPT_CONTENT="${PROMPT_CONTENT//\{\{PR_URL\}\}/${PR_URL}}" PROMPT_CONTENT="${PROMPT_CONTENT//\{\{PR_AUTHOR\}\}/${PR_AUTHOR}}" PROMPT_CONTENT="${PROMPT_CONTENT//\{\{PR_LABELS\}\}/${PR_LABELS:-none}}" - + # Write to temp file echo "$PROMPT_CONTENT" > "$output_file" - + # Verify prompt is not empty if [ ! -s "$output_file" ]; then echo "::error::Prompt file is empty after placeholder replacement" @@ -176,70 +223,77 @@ ${OVERRIDE_COMMENT} echo " 3. Placeholder replacement failed" return 1 fi - + echo "$output_file" } -# Run Gemini validation with retry logic -run_gemini() { +# Run ForgeCode validation with retry logic +run_forgecode() { local prompt_file="$1" local retry_count=0 local max_retries="${MAX_RETRIES:-2}" - + while [ $retry_count -le $max_retries ]; do - echo "Attempt $((retry_count + 1))/$((max_retries + 1)): Running Gemini validation..." - + echo "Attempt $((retry_count + 1))/$((max_retries + 1)): Running ForgeCode validation..." + # Debug: Check prompt file if [ ! -f "$prompt_file" ]; then echo "::error::Prompt file does not exist: $prompt_file" return 1 fi - + echo "Prompt file: $prompt_file" echo "Prompt file size: $(wc -c < "$prompt_file") bytes" echo "Prompt file lines: $(wc -l < "$prompt_file") lines" - + # Show detailed execution info - echo "๐Ÿค– Running Gemini CLI" + echo "๐Ÿค– Running ForgeCode CLI" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" - echo "Model: $GEMINI_MODEL" + echo "Provider: ${PROVIDER:-auto}" + echo "Model: ${MODEL}" echo "Prompt size: $(wc -c < "$prompt_file") bytes" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" echo "" - - # Run Gemini CLI and capture output + + # Configure ForgeCode with the specified model + if [ -n "${MODEL:-}" ]; then + echo "Configuring ForgeCode model: ${MODEL}" + forge config set --model "${MODEL}" 2>&1 || echo "Warning: Could not set model config" + fi + + # Run ForgeCode CLI and capture output set +e # Temporarily disable exit on error to capture exit code - - # Just use tee to show and save output - simple and effective - gemini -y -m "$GEMINI_MODEL" --prompt "$(cat "$prompt_file")" 2>&1 | tee /tmp/validation-full-output.md + + # Use -p flag for non-interactive mode with prompt from file + forge --verbose -p "$(cat "$prompt_file")" 2>&1 | tee /tmp/validation-full-output.md local exit_code=$? - + set -e # Re-enable exit on error - + echo "" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" - + if [ $exit_code -eq 0 ]; then - echo "โœ… Gemini CLI completed successfully (exit code: 0)" + echo "โœ… ForgeCode CLI completed successfully (exit code: 0)" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" return 0 else - echo "โŒ Gemini CLI failed (exit code: $exit_code)" + echo "โŒ ForgeCode CLI failed (exit code: $exit_code)" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" echo "" - + # Error details are shown above in the output echo "โš ๏ธ Check the output above for error details" - + # Check if it's a retryable error local is_retryable=false if [ -f /tmp/validation-full-output.md ] && grep -q -E "(429|503|timeout|rate limit)" /tmp/validation-full-output.md; then is_retryable=true fi - + if [ "$is_retryable" = true ]; then retry_count=$((retry_count + 1)) - + if [ $retry_count -le $max_retries ]; then wait_time=$((2 ** retry_count)) echo "โณ Rate limit or timeout detected. Retrying after ${wait_time} seconds..." @@ -255,11 +309,11 @@ run_gemini() { fi fi done - + return 1 } -# Extract validation report from Gemini output +# Extract validation report from ForgeCode output extract_report() { local full_output="$1" local report_file="/tmp/validation-report.md" @@ -390,9 +444,9 @@ main() { prompt_with_replacements=$(prepare_prompt "$actual_prompt_file") echo "Prompt prepared: $prompt_with_replacements" - - # Run Gemini validation - if ! run_gemini "$prompt_with_replacements"; then + + # Run ForgeCode validation + if ! run_forgecode "$prompt_with_replacements"; then echo "::error::Validation execution failed" # Set failed outputs using heredoc delimiter diff --git a/system-prompt.md b/system-prompt.md index 42e7cfc..4bb2817 100644 --- a/system-prompt.md +++ b/system-prompt.md @@ -48,27 +48,27 @@ # PR Validation Report ## Summary -[Brief 2-3 sentence overview of the PR and overall assessment] +{Brief 2-3 sentence overview of the PR and overall assessment} ## Critical Violations โŒ -[Must-fix issues that will fail the build] +{Must-fix issues that will fail the build} - **File**: `path/to/file.ts:42` -- **Issue**: [Clear description] -- **Why**: [Explanation of the violation] -- **Fix**: [Specific recommendation] +- **Issue**: {Clear description} +- **Why**: {Explanation of the violation} +- **Fix**: {Specific recommendation} ## Important Issues โš ๏ธ -[Should-fix issues that won't fail the build] +{Should-fix issues that won't fail the build} ## Suggestions โ„น๏ธ -[Nice-to-have improvements] +{Nice-to-have improvements} ## Validation Outcome - **Status**: PASS โœ… | FAIL โŒ -- **Critical Issues**: [count] -- **Important Issues**: [count] -- **Suggestions**: [count] +- **Critical Issues**: {count} +- **Important Issues**: {count} +- **Suggestions**: {count} ``` ## Handling Override Comments @@ -91,7 +91,7 @@ Understand what the user is asking for: **Use `create-memory-fragment` with these parameters:** - **workspaceId**: `60c10ca2-4115-4c1a-b6d7-04ac39fd3938` (Flowcore workspace) -- **title**: `Approved Deviation: [Brief description]` (be specific and clear) +- **title**: `Approved Deviation: {Brief description}` (be specific and clear) - **fragmentTypeId**: `b06897e0-c39e-486b-8a9b-aab0ea260694` (solution type) - **repository**: `usable-pr-validator` (or the appropriate repo name from PR context) - **tags**: Always include `["repo:", "deviation", "approved"]` plus any relevant tech tags @@ -102,22 +102,22 @@ Understand what the user is asking for: # Approved Deviation ## Standard Deviated From -[Clear description of what standard/rule is being deviated from] +{Clear description of what standard/rule is being deviated from} ## Reason for Deviation -[Business or technical justification provided by the user] +{Business or technical justification provided by the user} ## Conditions and Limitations -[Any constraints or conditions for this deviation] +{Any constraints or conditions for this deviation} ## Approval Details -- **PR**: #[PR_NUMBER] - [PR_URL] -- **Approved by**: @[COMMENT_AUTHOR] -- **Date**: [Current date in YYYY-MM-DD format] -- **Repository**: [repo name] +- **PR**: #{PR_NUMBER} - {PR_URL} +- **Approved by**: @{COMMENT_AUTHOR} +- **Date**: {Current date in YYYY-MM-DD format} +- **Repository**: {repo name} ## Related Context -[Any additional context from the PR or comment] +{Any additional context from the PR or comment} ``` ### 3. Include Fragment Link in Report @@ -129,10 +129,10 @@ After creating the fragment, **include it in your validation report**: A deviation from standards has been approved and documented: -- **Deviation**: [Brief description] -- **Justification**: [User's reason] -- **Documentation**: Fragment created - [Fragment title] (ID: [fragment-id]) -- **Approved by**: @[username] +- **Deviation**: {Brief description} +- **Justification**: {User's reason} +- **Documentation**: Fragment created - {Fragment title} (ID: {fragment-id}) +- **Approved by**: @{username} This deviation has been recorded in the knowledge base for future reference. ``` @@ -193,9 +193,12 @@ After documenting the deviation: - Identify the type of change (feature, fix, refactor, etc.) 2. **Analyze Changes** - - Get the diff: `git --no-pager diff origin/{base_branch}...{head_branch}` - - Identify all changed files - - Understand the scope and impact + - Review the changed files summary below: + + {{FILE_STATS}} + + - Use git commands to examine specific files: `git --no-pager diff origin/{base_branch}...{head_branch} -- ` + - Understand the scope and impact of changes **โš ๏ธ HANDLING GIT DIFF ERRORS:** diff --git a/test-local.sh b/test-local.sh new file mode 100755 index 0000000..326417c --- /dev/null +++ b/test-local.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Local testing script for PR validation +# This allows you to test the validator locally before pushing to GitHub + +echo "๐Ÿงช Usable PR Validator - Local Test" +echo "====================================" +echo "" + +# Check required environment variables +MISSING_VARS=() + +if [ -z "${OPENROUTER_API_KEY:-}${ANTHROPIC_API_KEY:-}${OPENAI_API_KEY:-}" ]; then + MISSING_VARS+=("At least one of: OPENROUTER_API_KEY, ANTHROPIC_API_KEY, OPENAI_API_KEY") +fi + +if [ -z "${USABLE_API_TOKEN:-}" ]; then + MISSING_VARS+=("USABLE_API_TOKEN") +fi + +if [ ${#MISSING_VARS[@]} -gt 0 ]; then + echo "โŒ Missing required environment variables:" + for var in "${MISSING_VARS[@]}"; do + echo " - $var" + done + echo "" + echo "Please set these in your environment or .env file" + echo "" + echo "Example:" + echo " export OPENROUTER_API_KEY='your-key-here'" + echo " export USABLE_API_TOKEN='your-token-here'" + exit 1 +fi + +# Set default values for local testing +export PROVIDER="${PROVIDER:-auto}" +export MODEL="${MODEL:-openai/gpt-oss-120b}" +export WORKSPACE_ID="${WORKSPACE_ID:-60c10ca2-4115-4c1a-b6d7-04ac39fd3938}" # Flowcore workspace +export MCP_SERVER_URL="${MCP_SERVER_URL:-https://usable.dev/api/mcp}" + +# Git configuration (you can override these) +export BASE_BRANCH="${BASE_BRANCH:-main}" +export HEAD_BRANCH="${HEAD_BRANCH:-$(git rev-parse --abbrev-ref HEAD)}" + +# PR metadata (simulated for local testing) +export PR_NUMBER="${PR_NUMBER:-0}" +export PR_TITLE="${PR_TITLE:-Local Test: $(git log -1 --pretty=%s)}" +export PR_DESCRIPTION="${PR_DESCRIPTION:-Testing PR validation locally}" +export PR_URL="${PR_URL:-https://github.com/local/test}" +export PR_AUTHOR="${PR_AUTHOR:-$(git config user.name)}" +export PR_LABELS="${PR_LABELS:-test,local}" + +# Other settings +export USE_DYNAMIC_PROMPTS="${USE_DYNAMIC_PROMPTS:-false}" +export PROMPT_FILE="${PROMPT_FILE:-./system-prompt.md}" +export MAX_RETRIES="${MAX_RETRIES:-2}" +export ALLOW_WEB_FETCH="${ALLOW_WEB_FETCH:-false}" +# Don't export these if they're empty - ForgeCode treats empty strings as file paths +if [ -n "${OVERRIDE_COMMENT:-}" ]; then + export OVERRIDE_COMMENT +fi +if [ -n "${COMMENT_AUTHOR:-}" ]; then + export COMMENT_AUTHOR +fi + +# GitHub Actions environment variables (simulated) +export GITHUB_OUTPUT="${GITHUB_OUTPUT:-/tmp/github-output.txt}" +export GITHUB_ENV="${GITHUB_ENV:-/tmp/github-env.txt}" +touch "$GITHUB_OUTPUT" "$GITHUB_ENV" + +echo "๐Ÿ“‹ Test Configuration" +echo "====================" +echo "Provider: $PROVIDER" +echo "Model: $MODEL" +echo "Base Branch: $BASE_BRANCH" +echo "Head Branch: $HEAD_BRANCH" +echo "Workspace: $WORKSPACE_ID" +echo "" + +# Check if ForgeCode is installed +if ! command -v forge &> /dev/null; then + echo "โš ๏ธ ForgeCode CLI not found. Installing..." + npm install -g forgecode@latest + echo "โœ… ForgeCode installed" + echo "" +fi + +# Verify git refs are available +echo "๐Ÿ” Verifying Git Setup" +echo "=====================" +if ! git rev-parse "origin/$BASE_BRANCH" >/dev/null 2>&1; then + echo "โš ๏ธ Base branch origin/$BASE_BRANCH not found. Fetching..." + git fetch origin "$BASE_BRANCH" +fi + +if ! git rev-parse "$HEAD_BRANCH" >/dev/null 2>&1; then + echo "โŒ Head branch $HEAD_BRANCH not found" + echo "Current branch: $(git rev-parse --abbrev-ref HEAD)" + exit 1 +fi + +echo "โœ… Git refs verified" +echo "" + +# Run provider setup +echo "๐Ÿ” Setting up Provider" +echo "=====================" +./scripts/setup-provider.sh +echo "" + +# Run MCP setup +echo "๐Ÿ”— Setting up MCP" +echo "================" +# For local testing, MCP_URL is already set and USABLE_API_TOKEN is in the environment +MCP_URL="${MCP_SERVER_URL}" ./scripts/setup-mcp.sh +echo "" + +# Check if we should fetch dynamic prompts +if [ "$USE_DYNAMIC_PROMPTS" = "true" ]; then + echo "๐Ÿ“ฅ Fetching Dynamic Prompts" + echo "==========================" + ./scripts/fetch-prompt.sh + echo "" +fi + +# Run validation +echo "๐Ÿš€ Running Validation" +echo "====================" +echo "" +./scripts/validate.sh + +# Show results +echo "" +echo "๐Ÿ“Š Validation Results" +echo "====================" +if [ -f "$GITHUB_OUTPUT" ]; then + echo "GitHub Outputs:" + cat "$GITHUB_OUTPUT" + echo "" +fi + +if [ -f /tmp/validation-report.md ]; then + echo "Validation Report:" + echo "==================" + cat /tmp/validation-report.md + echo "" +fi + +echo "โœ… Local test completed" +echo "" +echo "Artifacts:" +echo " - Full output: /tmp/validation-full-output.md" +echo " - Report: /tmp/validation-report.md" +echo " - GitHub outputs: $GITHUB_OUTPUT" From 0ea807a05eb93bcab83c85bff84725f6c5d03b2c Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Wed, 22 Oct 2025 22:49:13 +0100 Subject: [PATCH 02/41] fix: update default model references in action.yml, README.md, and workflows - Changed default model from 'openai/gpt-oss-120b' to 'anthropic/claude-haiku-4.5' across action.yml, README.md, and various workflow files. - Updated comments and documentation to reflect the new default model and its options for better clarity and user guidance. --- .github/workflows/comment-revalidation.yml | 4 ++-- .github/workflows/test-reusable-workflow.yml | 6 +++--- README.md | 6 +++--- action.yml | 4 ++-- test-local.sh | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/comment-revalidation.yml b/.github/workflows/comment-revalidation.yml index a31a23b..f725c3c 100644 --- a/.github/workflows/comment-revalidation.yml +++ b/.github/workflows/comment-revalidation.yml @@ -34,7 +34,7 @@ on: description: 'Model to use' required: false type: string - default: 'openai/gpt-oss-120b' + default: 'anthropic/claude-haiku-4.5' comment-title: description: 'Title for the validation comment' required: false @@ -285,7 +285,7 @@ jobs: workspace-id: ${{ inputs.workspace-id || '60c10ca2-4115-4c1a-b6d7-04ac39fd3938' }} base-ref: ${{ steps.base-ref.outputs.base-ref }} provider: ${{ inputs.provider || 'auto' }} - model: ${{ inputs.model || 'openai/gpt-oss-120b' }} + model: ${{ inputs.model || 'anthropic/claude-haiku-4.5' }} comment-title: ${{ inputs.comment-title || '๐Ÿ”„ Revalidation (Comment Triggered)' }} comment-mode: 'create' fail-on-critical: ${{ inputs.fail-on-critical || false }} diff --git a/.github/workflows/test-reusable-workflow.yml b/.github/workflows/test-reusable-workflow.yml index 5447aac..d7c8715 100644 --- a/.github/workflows/test-reusable-workflow.yml +++ b/.github/workflows/test-reusable-workflow.yml @@ -24,7 +24,7 @@ jobs: workspace-id: '60c10ca2-4115-4c1a-b6d7-04ac39fd3938' prompt-file: 'templates/basic-validation.md' provider: 'auto' - model: 'openai/gpt-oss-120b' + model: 'anthropic/claude-haiku-4.5' comment-title: '๐Ÿงช Reusable Workflow Test' fail-on-critical: false secrets: @@ -64,7 +64,7 @@ jobs: - Workspace ID: 60c10ca2-4115-4c1a-b6d7-04ac39fd3938 - Prompt File: templates/basic-validation.md - Provider: auto - - Model: openai/gpt-oss-120b + - Model: anthropic/claude-haiku-4.5 - Comment Title: ๐Ÿงช Reusable Workflow Test **Result**: ${{ needs.test-reusable.result }} @@ -78,7 +78,7 @@ jobs: uses: flowcore/usable-pr-validator/.github/workflows/comment-revalidation.yml@v1 with: workspace-id: 'their-workspace-uuid' - # Optional - defaults to auto and openai/gpt-oss-120b + # Optional - defaults to auto and anthropic/claude-haiku-4.5 # provider: 'openrouter' # or 'anthropic', 'openai', 'auto' # model: 'anthropic/claude-3.7-sonnet' # for higher quality secrets: diff --git a/README.md b/README.md index 37cbdd4..520f260 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ jobs: with: prompt-file: '.github/prompts/pr-validation.md' workspace-id: 'your-workspace-uuid' - # Optional: specify provider and model (defaults to auto-detect and openai/gpt-oss-120b) + # Optional: specify provider and model (defaults to auto-detect and anthropic/claude-haiku-4.5) # provider: 'openrouter' # or 'anthropic', 'openai', 'auto' # model: 'anthropic/claude-3.7-sonnet' # for higher quality env: @@ -119,7 +119,7 @@ export USABLE_API_TOKEN='your-usable-token' # Optional (can be set in the script or use defaults) export PROVIDER='auto' # or 'openrouter', 'anthropic', 'openai' -export MODEL='openai/gpt-oss-120b' # default model (fast and cost-effective) +export MODEL='anthropic/claude-haiku-4.5' # default model (fast and cost-effective) export WORKSPACE_ID='your-workspace-uuid' # Usable workspace export BASE_BRANCH='main' # base branch for diff export HEAD_BRANCH='feature-branch' # current branch @@ -161,7 +161,7 @@ The script will: | `workspace-id` | Usable workspace UUID (required - used to fetch MCP system prompt) | โœ“ | | | `merge-custom-prompt` | Merge fetched Usable prompt with custom `prompt-file` (only when both are provided) | | `true` | | `provider` | LLM provider: `openrouter`, `openai`, `anthropic`, or `auto` (auto-detect from env vars) | | `auto` | -| `model` | Model to use (e.g., `openai/gpt-oss-120b`, `anthropic/claude-3.7-sonnet`, `openai/gpt-4`, `meta-llama/llama-3.3-70b-instruct`) | | `openai/gpt-oss-120b` | +| `model` | Model to use (e.g., `anthropic/claude-haiku-4.5`, `anthropic/claude-3.7-sonnet`, `openai/gpt-4`, `meta-llama/llama-3.3-70b-instruct`) | | `anthropic/claude-haiku-4.5` | | `api-key-secret` | Name of secret containing API key | | `OPENROUTER_API_KEY` | | `mcp-server-url` | Usable MCP server URL | | `https://usable.dev/api/mcp` | | `mcp-token-secret` | Secret name for Usable API token | | `USABLE_API_TOKEN` | diff --git a/action.yml b/action.yml index 000d04d..c92e4fe 100644 --- a/action.yml +++ b/action.yml @@ -32,9 +32,9 @@ inputs: required: false default: 'auto' model: - description: 'Model to use (e.g., openai/gpt-oss-120b, anthropic/claude-3.7-sonnet, openai/gpt-4, meta-llama/llama-3.3-70b-instruct)' + description: 'Model to use (e.g., anthropic/claude-haiku-4.5, anthropic/claude-3.7-sonnet, openai/gpt-4, meta-llama/llama-3.3-70b-instruct)' required: false - default: 'openai/gpt-oss-120b' + default: 'anthropic/claude-haiku-4.5' api-key-secret: description: 'Name of secret containing API key (OPENROUTER_API_KEY, OPENAI_API_KEY, or ANTHROPIC_API_KEY)' required: false diff --git a/test-local.sh b/test-local.sh index 326417c..2cae3db 100755 --- a/test-local.sh +++ b/test-local.sh @@ -35,7 +35,7 @@ fi # Set default values for local testing export PROVIDER="${PROVIDER:-auto}" -export MODEL="${MODEL:-openai/gpt-oss-120b}" +export MODEL="${MODEL:-anthropic/claude-haiku-4.5}" export WORKSPACE_ID="${WORKSPACE_ID:-60c10ca2-4115-4c1a-b6d7-04ac39fd3938}" # Flowcore workspace export MCP_SERVER_URL="${MCP_SERVER_URL:-https://usable.dev/api/mcp}" From cf948dbdfa776bdd0d503ea9e6c4b2ea80dd55e9 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Wed, 22 Oct 2025 23:08:30 +0100 Subject: [PATCH 03/41] refactor(setup-mcp.sh): streamline MCP server configuration for ForgeCode - Removed the creation of a temporary configuration file and instead directly added the Usable MCP server using the official @usabledev/mcp-server package. - Enhanced the script to remove any existing usable MCP server to avoid duplicates before adding the new server. - Updated output messages to reflect the new configuration process and provide clearer feedback on the setup status. --- scripts/setup-mcp.sh | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index 615a1c1..b2ad9d4 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -27,31 +27,26 @@ if [ -z "$MCP_URL" ]; then exit 1 fi -# Create ForgeCode MCP configuration -# ForgeCode looks for MCP config in forge.yaml or can use environment variables -cat > /tmp/forge-mcp-config.yaml <> "$GITHUB_ENV" -fi +# Remove any existing usable MCP server first (to avoid duplicates) +forge mcp remove usable 2>/dev/null || echo " No existing usable server to remove" + +# Add Usable MCP server using the official @usabledev/mcp-server package +# This uses stdio transport and handles authentication via environment variables +echo "Adding Usable MCP server to ForgeCode..." +forge mcp add usable "npx" \ + --transport stdio \ + --scope local \ + --args "@usabledev/mcp-server@latest" \ + --args "server" \ + --env "USABLE_API_TOKEN=$MCP_TOKEN" \ + --env "USABLE_BASE_URL=$MCP_URL" 2>&1 echo "โœ… MCP server configured for ForgeCode" -echo " URL: $MCP_URL" -echo " Config file: /tmp/forge-mcp-config.yaml" +echo " Package: @usabledev/mcp-server@latest" +echo " Base URL: $MCP_URL" -# Debug: Show settings file content (mask token) -echo " Configuration preview:" -cat /tmp/forge-mcp-config.yaml | sed 's/Bearer [^"]*$/Bearer ***MASKED***/g' | sed 's/^/ /' +# List configured servers for verification +echo " Configured MCP servers:" +forge mcp list 2>&1 | sed 's/^/ /' || echo " (Unable to list servers)" echo "::endgroup::" From fca53e9c23492a55acfa3b93da4e4bf4d70a4974 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Wed, 22 Oct 2025 23:21:28 +0100 Subject: [PATCH 04/41] feat(validation): implement JSON output schema and markdown report generation - Introduced a new JSON schema for the PR validation report, detailing required fields such as summary, critical violations, important issues, suggestions, and validation outcome. - Updated the system prompt to specify the JSON file writing requirements and the structure of the output. - Added a script to construct a markdown report from the generated JSON file, enhancing the clarity and usability of validation results. - Improved the validation script to handle JSON output and ensure proper extraction of report data, including critical and important issues. --- schemas/validation-report.schema.json | 169 ++++++++++++++++++++++ scripts/construct-markdown.sh | 194 ++++++++++++++++++++++++++ scripts/validate.sh | 97 +++++++++---- system-prompt.md | 113 ++++++++------- 4 files changed, 488 insertions(+), 85 deletions(-) create mode 100644 schemas/validation-report.schema.json create mode 100755 scripts/construct-markdown.sh diff --git a/schemas/validation-report.schema.json b/schemas/validation-report.schema.json new file mode 100644 index 0000000..95c9b31 --- /dev/null +++ b/schemas/validation-report.schema.json @@ -0,0 +1,169 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "PR Validation Report", + "description": "Structured output schema for PR validation results", + "type": "object", + "required": ["summary", "criticalViolations", "importantIssues", "suggestions", "validationOutcome"], + "properties": { + "summary": { + "type": "string", + "description": "Brief 2-3 sentence overview of the PR and overall assessment" + }, + "criticalViolations": { + "type": "array", + "description": "Must-fix issues that will fail the build", + "items": { + "type": "object", + "required": ["file", "issue", "why", "fix"], + "properties": { + "file": { + "type": "string", + "description": "File path and optional line number (e.g., 'path/to/file.ts:42')" + }, + "issue": { + "type": "string", + "description": "Clear description of the violation" + }, + "why": { + "type": "string", + "description": "Explanation of why this violates standards" + }, + "fix": { + "type": "string", + "description": "Specific recommendation on how to fix it" + }, + "code": { + "type": "string", + "description": "Optional: The problematic code snippet" + } + } + } + }, + "importantIssues": { + "type": "array", + "description": "Should-fix issues that won't fail the build", + "items": { + "type": "object", + "required": ["file", "issue", "why", "fix"], + "properties": { + "file": { + "type": "string", + "description": "File path and optional line number" + }, + "issue": { + "type": "string", + "description": "Clear description of the issue" + }, + "why": { + "type": "string", + "description": "Explanation of the issue" + }, + "fix": { + "type": "string", + "description": "Recommended improvement" + }, + "code": { + "type": "string", + "description": "Optional: The problematic code snippet" + } + } + } + }, + "suggestions": { + "type": "array", + "description": "Nice-to-have improvements", + "items": { + "type": "object", + "required": ["title", "description"], + "properties": { + "title": { + "type": "string", + "description": "Brief title of the suggestion" + }, + "description": { + "type": "string", + "description": "Detailed explanation of the suggestion" + }, + "file": { + "type": "string", + "description": "Optional: Related file path" + } + } + } + }, + "validationOutcome": { + "type": "object", + "required": ["status", "criticalIssuesCount", "importantIssuesCount", "suggestionsCount"], + "properties": { + "status": { + "type": "string", + "enum": ["PASS", "FAIL"], + "description": "Overall validation status" + }, + "criticalIssuesCount": { + "type": "integer", + "minimum": 0, + "description": "Number of critical violations found" + }, + "importantIssuesCount": { + "type": "integer", + "minimum": 0, + "description": "Number of important issues found" + }, + "suggestionsCount": { + "type": "integer", + "minimum": 0, + "description": "Number of suggestions provided" + }, + "rationale": { + "type": "string", + "description": "Optional: Brief explanation of the validation outcome" + } + } + }, + "overrideApplied": { + "type": "object", + "description": "Optional: Information about applied overrides (for comment-triggered revalidation)", + "properties": { + "deviation": { + "type": "string", + "description": "What deviation was approved" + }, + "justification": { + "type": "string", + "description": "Why the deviation was necessary" + }, + "fragmentId": { + "type": "string", + "description": "ID of the created deviation fragment" + }, + "fragmentTitle": { + "type": "string", + "description": "Title of the created deviation fragment" + }, + "approvedBy": { + "type": "string", + "description": "Username who approved the deviation" + } + } + }, + "metadata": { + "type": "object", + "description": "Optional: Additional metadata about the validation", + "properties": { + "triggeredBy": { + "type": "string", + "description": "How this validation was triggered (e.g., 'PR opened', 'Comment by @user')" + }, + "standardsChecked": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of standards/patterns checked" + } + } + } + } +} + diff --git a/scripts/construct-markdown.sh b/scripts/construct-markdown.sh new file mode 100755 index 0000000..46206db --- /dev/null +++ b/scripts/construct-markdown.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Construct markdown comment from JSON validation report +# Usage: construct-markdown.sh + +JSON_FILE="${1:-/tmp/validation-report.json}" +OUTPUT_FILE="${2:-/tmp/validation-report.md}" + +if [ ! -f "$JSON_FILE" ]; then + echo "::error::JSON report file not found: $JSON_FILE" + exit 1 +fi + +# Verify JSON is valid +if ! jq empty "$JSON_FILE" 2>/dev/null; then + echo "::error::Invalid JSON in report file" + exit 1 +fi + +# Start constructing the markdown report +{ + echo "# PR Validation Report" + echo "" + + # Summary + echo "## Summary" + jq -r '.summary' "$JSON_FILE" + echo "" + + # Override information (if present) + if jq -e '.overrideApplied' "$JSON_FILE" > /dev/null 2>&1; then + echo "## Override Applied ๐Ÿ”„" + echo "" + echo "A deviation from standards has been approved and documented:" + echo "" + echo "- **Deviation**: $(jq -r '.overrideApplied.deviation // "N/A"' "$JSON_FILE")" + echo "- **Justification**: $(jq -r '.overrideApplied.justification // "N/A"' "$JSON_FILE")" + + if jq -e '.overrideApplied.fragmentId' "$JSON_FILE" > /dev/null 2>&1; then + fragment_id=$(jq -r '.overrideApplied.fragmentId' "$JSON_FILE") + fragment_title=$(jq -r '.overrideApplied.fragmentTitle // "Deviation"' "$JSON_FILE") + echo "- **Documentation**: Fragment created - ${fragment_title} (ID: ${fragment_id})" + fi + + if jq -e '.overrideApplied.approvedBy' "$JSON_FILE" > /dev/null 2>&1; then + approved_by=$(jq -r '.overrideApplied.approvedBy' "$JSON_FILE") + echo "- **Approved by**: @${approved_by}" + fi + + echo "" + echo "This deviation has been recorded in the knowledge base for future reference." + echo "" + fi + + # Critical Violations + echo "## Critical Violations โŒ" + echo "" + + critical_count=$(jq '.criticalViolations | length' "$JSON_FILE") + + if [ "$critical_count" -eq 0 ]; then + echo "None found." + else + # Iterate through critical violations + jq -c '.criticalViolations[]' "$JSON_FILE" | while read -r violation; do + file=$(echo "$violation" | jq -r '.file') + issue=$(echo "$violation" | jq -r '.issue') + why=$(echo "$violation" | jq -r '.why') + fix=$(echo "$violation" | jq -r '.fix') + code=$(echo "$violation" | jq -r '.code // empty') + + echo "- **File**: \`$file\`" + echo " - **Issue**: $issue" + echo " - **Why**: $why" + echo " - **Fix**: $fix" + + if [ -n "$code" ]; then + echo " - **Code**:" + echo ' ```' + echo "$code" | sed 's/^/ /' + echo ' ```' + fi + echo "" + done + fi + + echo "" + + # Important Issues + echo "## Important Issues โš ๏ธ" + echo "" + + important_count=$(jq '.importantIssues | length' "$JSON_FILE") + + if [ "$important_count" -eq 0 ]; then + echo "None found." + else + jq -c '.importantIssues[]' "$JSON_FILE" | while read -r issue; do + file=$(echo "$issue" | jq -r '.file') + issue_text=$(echo "$issue" | jq -r '.issue') + why=$(echo "$issue" | jq -r '.why') + fix=$(echo "$issue" | jq -r '.fix') + code=$(echo "$issue" | jq -r '.code // empty') + + echo "- **File**: \`$file\`" + echo " - **Issue**: $issue_text" + echo " - **Why**: $why" + echo " - **Fix**: $fix" + + if [ -n "$code" ]; then + echo " - **Code**:" + echo ' ```' + echo "$code" | sed 's/^/ /' + echo ' ```' + fi + echo "" + done + fi + + echo "" + + # Suggestions + echo "## Suggestions โ„น๏ธ" + echo "" + + suggestions_count=$(jq '.suggestions | length' "$JSON_FILE") + + if [ "$suggestions_count" -eq 0 ]; then + echo "None found." + else + jq -c '.suggestions[]' "$JSON_FILE" | while read -r suggestion; do + title=$(echo "$suggestion" | jq -r '.title') + description=$(echo "$suggestion" | jq -r '.description') + file=$(echo "$suggestion" | jq -r '.file // empty') + + echo "- **$title**: $description" + + if [ -n "$file" ]; then + echo " - File: \`$file\`" + fi + echo "" + done + fi + + echo "" + + # Validation Outcome + echo "## Validation Outcome" + echo "" + + status=$(jq -r '.validationOutcome.status' "$JSON_FILE") + critical=$(jq -r '.validationOutcome.criticalIssuesCount' "$JSON_FILE") + important=$(jq -r '.validationOutcome.importantIssuesCount' "$JSON_FILE") + suggestions=$(jq -r '.validationOutcome.suggestionsCount' "$JSON_FILE") + + if [ "$status" = "PASS" ]; then + echo "- **Status**: PASS โœ…" + else + echo "- **Status**: FAIL โŒ" + fi + + echo "- **Critical Issues**: $critical" + echo "- **Important Issues**: $important" + echo "- **Suggestions**: $suggestions" + + # Add rationale if present + if jq -e '.validationOutcome.rationale' "$JSON_FILE" > /dev/null 2>&1; then + rationale=$(jq -r '.validationOutcome.rationale' "$JSON_FILE") + echo "" + echo "**Rationale**: $rationale" + fi + + # Add metadata if present + if jq -e '.metadata.triggeredBy' "$JSON_FILE" > /dev/null 2>&1; then + triggered_by=$(jq -r '.metadata.triggeredBy' "$JSON_FILE") + echo "" + echo "- **Triggered by**: $triggered_by" + fi + + if jq -e '.metadata.standardsChecked' "$JSON_FILE" > /dev/null 2>&1; then + echo "" + echo "### Standards Checked" + echo "" + jq -r '.metadata.standardsChecked[]' "$JSON_FILE" | while read -r standard; do + echo "- $standard" + done + fi + +} > "$OUTPUT_FILE" + +echo "โœ… Markdown report constructed: $OUTPUT_FILE" +echo " Lines: $(wc -l < "$OUTPUT_FILE")" + diff --git a/scripts/validate.sh b/scripts/validate.sh index 4db10e0..63b02af 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -265,7 +265,8 @@ run_forgecode() { set +e # Temporarily disable exit on error to capture exit code # Use -p flag for non-interactive mode with prompt from file - forge --verbose -p "$(cat "$prompt_file")" 2>&1 | tee /tmp/validation-full-output.md + # Note: Removed --verbose to get clean output without debugging info + forge -p "$(cat "$prompt_file")" 2>&1 | tee /tmp/validation-full-output.md local exit_code=$? set -e # Re-enable exit on error @@ -317,42 +318,51 @@ run_forgecode() { extract_report() { local full_output="$1" local report_file="/tmp/validation-report.md" + local clean_output="/tmp/validation-clean-output.md" + + # First, strip ANSI escape codes from the output + # This handles color codes and formatting that might interfere with pattern matching + if command -v sed &> /dev/null; then + sed 's/\x1b\[[0-9;]*m//g' "$full_output" > "$clean_output" + else + cp "$full_output" "$clean_output" + fi # Strategy 1: Look for "# PR Validation Report" header - if grep -q "# PR Validation Report" "$full_output"; then + if grep -q "# PR Validation Report" "$clean_output"; then echo "Extracting report using Strategy 1: PR Validation Report header" - sed -n '/# PR Validation Report/,$p' "$full_output" > "$report_file" + sed -n '/# PR Validation Report/,$p' "$clean_output" > "$report_file" return 0 fi # Strategy 2: Look for "## Summary" section - if grep -q "## Summary" "$full_output"; then + if grep -q "## Summary" "$clean_output"; then echo "Extracting report using Strategy 2: Summary section" - sed -n '/## Summary/,$p' "$full_output" > "$report_file" + sed -n '/## Summary/,$p' "$clean_output" > "$report_file" echo "# PR Validation Report" | cat - "$report_file" > /tmp/temp && mv /tmp/temp "$report_file" return 0 fi # Strategy 3: Look for "## Critical Violations" section - if grep -q "## Critical Violations" "$full_output"; then + if grep -q "## Critical Violations" "$clean_output"; then echo "Extracting report using Strategy 3: Critical Violations section" - sed -n '/## Critical Violations/,$p' "$full_output" > "$report_file" + sed -n '/## Critical Violations/,$p' "$clean_output" > "$report_file" echo "# PR Validation Report" | cat - "$report_file" > /tmp/temp && mv /tmp/temp "$report_file" return 0 fi # Strategy 4: Use full output with warning echo "::warning::Could not find report markers. Using full output." - echo "::group::Gemini Full Output (first 50 lines)" - head -50 "$full_output" || echo "Could not read output file" + echo "::group::ForgeCode Full Output (first 50 lines)" + head -50 "$clean_output" || echo "Could not read output file" echo "::endgroup::" - if [ ! -f "$full_output" ]; then - echo "::error::Full output file does not exist: $full_output" + if [ ! -f "$clean_output" ]; then + echo "::error::Clean output file does not exist: $clean_output" return 1 fi - cp "$full_output" "$report_file" + cp "$clean_output" "$report_file" if [ ! -f "$report_file" ]; then echo "::error::Failed to create report file: $report_file" @@ -363,16 +373,19 @@ extract_report() { return 0 } -# Parse validation results and set GitHub outputs +# Parse validation results from JSON and set GitHub outputs parse_results() { - local report_file="$1" + local json_file="/tmp/validation-report.json" - # Check for PASS/FAIL status + # Extract values from JSON local validation_status local validation_passed local critical_issues - if grep -q -i "Status.*PASS" "$report_file" || grep -q "โœ…" "$report_file"; then + # Get status from JSON + local status=$(jq -r '.validationOutcome.status' "$json_file") + + if [ "$status" = "PASS" ]; then validation_status="passed" validation_passed="true" else @@ -380,10 +393,8 @@ parse_results() { validation_passed="false" fi - # Count critical issues (looking for unchecked critical violations) - # Strip any whitespace/newlines and ensure we get a clean integer - critical_issues=$(grep -c "^- \[ \] \*\*" "$report_file" 2>/dev/null || echo "0") - critical_issues=$(echo "$critical_issues" | tr -d '\n\r' | tr -d ' ') + # Get critical issues count from JSON + critical_issues=$(jq -r '.validationOutcome.criticalIssuesCount' "$json_file") # Ensure we have a valid integer (default to 0 if empty or invalid) if ! [[ "$critical_issues" =~ ^[0-9]+$ ]]; then @@ -466,30 +477,54 @@ main() { exit 1 fi - # Extract report from output - echo "::group::Extracting validation report" - echo "Full output file: /tmp/validation-full-output.md" - if [ -f "/tmp/validation-full-output.md" ]; then - echo "โœ… Full output file exists ($(wc -l < /tmp/validation-full-output.md) lines)" - else - echo "::error::Full output file does not exist!" + # Check if JSON report was created + echo "::group::Processing validation report" + + if [ ! -f "/tmp/validation-report.json" ]; then + echo "::error::JSON report file not created by ForgeCode" + echo "Expected file: /tmp/validation-report.json" + echo "" + echo "::group::ForgeCode output (for debugging)" + if [ -f "/tmp/validation-full-output.md" ]; then + head -100 "/tmp/validation-full-output.md" || echo "Could not read output file" + fi + echo "::endgroup::" + echo "::endgroup::" exit 1 fi - if ! extract_report "/tmp/validation-full-output.md"; then - echo "::error::Failed to extract validation report" + echo "โœ… JSON report found" + + # Validate JSON + if ! jq empty "/tmp/validation-report.json" 2>/dev/null; then + echo "::error::Invalid JSON in report file" + echo "::group::JSON content" + cat "/tmp/validation-report.json" || echo "Could not read JSON file" + echo "::endgroup::" echo "::endgroup::" exit 1 fi + + echo "โœ… JSON is valid" + + # Construct markdown report from JSON + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + if ! "$SCRIPT_DIR/construct-markdown.sh" "/tmp/validation-report.json" "/tmp/validation-report.md"; then + echo "::error::Failed to construct markdown from JSON" + echo "::endgroup::" + exit 1 + fi + + echo "โœ… Markdown report constructed" echo "::endgroup::" # Parse results and set outputs echo "Parsing validation results..." # Set GitHub outputs and get results - if [ -f "/tmp/validation-report.md" ]; then + if [ -f "/tmp/validation-report.json" ]; then # parse_results writes to GITHUB_OUTPUT and returns display values - results=$(parse_results "/tmp/validation-report.md") + results=$(parse_results) # Extract values for display (pipe-separated format) IFS='|' read -r validation_status validation_passed critical_issues <<< "$results" diff --git a/system-prompt.md b/system-prompt.md index 4bb2817..ada1fc2 100644 --- a/system-prompt.md +++ b/system-prompt.md @@ -29,47 +29,65 @@ ## Output Format Requirements -### CRITICAL: Start Your Output - -**START YOUR OUTPUT DIRECTLY WITH:** `# PR Validation Report` - -**DO NOT** include in your output: - -- Your thinking process -- Standards content you fetched from Usable -- Git command outputs -- Tool execution logs -- Debug information -- Preamble or explanation of what you're about to do - -### Report Structure - -```markdown -# PR Validation Report - -## Summary -{Brief 2-3 sentence overview of the PR and overall assessment} - -## Critical Violations โŒ -{Must-fix issues that will fail the build} - -- **File**: `path/to/file.ts:42` -- **Issue**: {Clear description} -- **Why**: {Explanation of the violation} -- **Fix**: {Specific recommendation} - -## Important Issues โš ๏ธ -{Should-fix issues that won't fail the build} +### CRITICAL: Write JSON File + +**Write a JSON file to `/tmp/validation-report.json` following this schema:** + +```json +{ + "summary": "string (required) - Brief 2-3 sentence overview", + "criticalViolations": [ + { + "file": "string (required) - path/to/file.ts:42", + "issue": "string (required) - Clear description", + "why": "string (required) - Why this violates standards", + "fix": "string (required) - How to fix it", + "code": "string (optional) - Problematic code snippet" + } + ], + "importantIssues": [ + { + "file": "string (required)", + "issue": "string (required)", + "why": "string (required)", + "fix": "string (required)", + "code": "string (optional)" + } + ], + "suggestions": [ + { + "title": "string (required) - Brief title", + "description": "string (required) - Detailed explanation", + "file": "string (optional) - Related file path" + } + ], + "validationOutcome": { + "status": "string (required) - PASS or FAIL", + "criticalIssuesCount": "number (required)", + "importantIssuesCount": "number (required)", + "suggestionsCount": "number (required)", + "rationale": "string (optional) - Brief explanation" + }, + "overrideApplied": { + "deviation": "string (optional) - What was approved", + "justification": "string (optional) - Why it was needed", + "fragmentId": "string (optional) - Created fragment ID", + "fragmentTitle": "string (optional) - Fragment title", + "approvedBy": "string (optional) - Username" + }, + "metadata": { + "triggeredBy": "string (optional)", + "standardsChecked": ["array of strings (optional)"] + } +} +``` -## Suggestions โ„น๏ธ -{Nice-to-have improvements} +**Key Requirements**: -## Validation Outcome -- **Status**: PASS โœ… | FAIL โŒ -- **Critical Issues**: {count} -- **Important Issues**: {count} -- **Suggestions**: {count} -``` +- Arrays can be empty: `[]` +- Status must be exactly "PASS" or "FAIL" +- Counts must match array lengths +- Include line numbers in file paths when applicable ## Handling Override Comments @@ -120,22 +138,9 @@ Understand what the user is asking for: {Any additional context from the PR or comment} ``` -### 3. Include Fragment Link in Report +### 3. Include Override Information in JSON -After creating the fragment, **include it in your validation report**: - -```markdown -## Override Applied - -A deviation from standards has been approved and documented: - -- **Deviation**: {Brief description} -- **Justification**: {User's reason} -- **Documentation**: Fragment created - {Fragment title} (ID: {fragment-id}) -- **Approved by**: @{username} - -This deviation has been recorded in the knowledge base for future reference. -``` +After creating the fragment, include the `overrideApplied` object in your JSON file with the fragment details. The markdown report will automatically generate an "Override Applied" section. ### 4. Adjust Validation Accordingly From b32179f7de92d863cc0069d639922079d3467d82 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Wed, 22 Oct 2025 23:25:26 +0100 Subject: [PATCH 05/41] chore(setup-mcp.sh, .gitignore): update MCP server configuration and ignore JSON files - Modified the setup-mcp.sh script to use HTTP transport with an Authorization header for adding the Usable MCP server, enhancing security and configuration clarity. - Updated output messages to reflect the new transport method and configuration details. - Added .mcp.json to .gitignore to prevent tracking of sensitive configuration files. --- .gitignore | 1 + scripts/setup-mcp.sh | 18 ++++++++---------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 44e071d..4904c67 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .DS_Store .AppleDouble .LSOverride +.mcp.json # Secrets and credentials *.json.key diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index b2ad9d4..97721af 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -30,19 +30,17 @@ fi # Remove any existing usable MCP server first (to avoid duplicates) forge mcp remove usable 2>/dev/null || echo " No existing usable server to remove" -# Add Usable MCP server using the official @usabledev/mcp-server package -# This uses stdio transport and handles authentication via environment variables +# Add Usable MCP server using HTTP transport with Authorization header +# ForgeCode requires JSON configuration to pass HTTP headers echo "Adding Usable MCP server to ForgeCode..." -forge mcp add usable "npx" \ - --transport stdio \ - --scope local \ - --args "@usabledev/mcp-server@latest" \ - --args "server" \ - --env "USABLE_API_TOKEN=$MCP_TOKEN" \ - --env "USABLE_BASE_URL=$MCP_URL" 2>&1 +echo " URL: $MCP_URL" +echo " Auth: Bearer token (configured)" + +# Use forge mcp add-json to configure HTTP transport with Authorization header +forge mcp add-json "usable" "{\"url\":\"$MCP_URL\",\"headers\":{\"Authorization\":\"Bearer $MCP_TOKEN\"}}" 2>&1 echo "โœ… MCP server configured for ForgeCode" -echo " Package: @usabledev/mcp-server@latest" +echo " Transport: HTTP with Authorization header" echo " Base URL: $MCP_URL" # List configured servers for verification From 31400aa1710c2ca72c031d553fa1396baee032e1 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Wed, 22 Oct 2025 23:47:07 +0100 Subject: [PATCH 06/41] refactor(action.yml, README.md, test-local.sh, scripts): update MCP server configuration and environment variable handling - Removed deprecated inputs for MCP server URL and token from action.yml, simplifying the configuration. - Updated README.md to reflect changes in environment variable requirements for Usable API integration. - Modified test-local.sh to use the new environment variable structure for local testing. - Adjusted scripts to fetch the Usable API token and URL directly from the environment, enhancing clarity and usability. - Improved output messages in setup-mcp.sh to provide better feedback on the configuration process. --- README.md | 15 +++++++++++---- action.yml | 15 ++++----------- scripts/fetch-prompt.sh | 32 ++++++++++++++++---------------- scripts/setup-mcp.sh | 36 +++++++++++++----------------------- test-local.sh | 23 +++++++++++++---------- 5 files changed, 57 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 520f260..c791949 100644 --- a/README.md +++ b/README.md @@ -118,8 +118,9 @@ export OPENAI_API_KEY='your-openai-key' # For OpenAI export USABLE_API_TOKEN='your-usable-token' # Optional (can be set in the script or use defaults) +export USABLE_URL='https://usable.dev' # Usable instance URL (default) export PROVIDER='auto' # or 'openrouter', 'anthropic', 'openai' -export MODEL='anthropic/claude-haiku-4.5' # default model (fast and cost-effective) +export MODEL='anthropic/claude-haiku-4.5' # default model (fast and cost-effective) export WORKSPACE_ID='your-workspace-uuid' # Usable workspace export BASE_BRANCH='main' # base branch for diff export HEAD_BRANCH='feature-branch' # current branch @@ -163,8 +164,6 @@ The script will: | `provider` | LLM provider: `openrouter`, `openai`, `anthropic`, or `auto` (auto-detect from env vars) | | `auto` | | `model` | Model to use (e.g., `anthropic/claude-haiku-4.5`, `anthropic/claude-3.7-sonnet`, `openai/gpt-4`, `meta-llama/llama-3.3-70b-instruct`) | | `anthropic/claude-haiku-4.5` | | `api-key-secret` | Name of secret containing API key | | `OPENROUTER_API_KEY` | -| `mcp-server-url` | Usable MCP server URL | | `https://usable.dev/api/mcp` | -| `mcp-token-secret` | Secret name for Usable API token | | `USABLE_API_TOKEN` | | `fail-on-critical` | Fail build on critical violations | | `true` | | `comment-mode` | PR comment behavior (`update`/`new`/`none`) | | `update` | | `comment-title` | Title for PR comment (for multi-stage validation) | | `Automated Standards Validation` | @@ -175,7 +174,15 @@ The script will: | `head-ref` | Head reference for diff comparison | | PR head branch | | `allow-web-fetch` | Allow AI to use web_fetch tool for external resources (security consideration) | | `false` | -> **Note**: You must set the `USABLE_API_TOKEN` secret (or the custom secret name specified in `mcp-token-secret`). Usable MCP integration is required for this action. +#### Environment Variables (Required in Workflow) + +| Variable | Description | Required | +|----------|-------------|----------| +| `OPENROUTER_API_KEY` | Your OpenRouter API key (or `ANTHROPIC_API_KEY`/`OPENAI_API_KEY`) | โœ“ | +| `USABLE_API_TOKEN` | Your Usable API token for MCP integration | โœ“ | +| `USABLE_URL` | Usable instance URL (defaults to `https://usable.dev`) | | + +> **Note**: You must set the `USABLE_API_TOKEN` secret. Usable MCP integration is required for this action. ### ๐Ÿง  System Prompts (Automatic) diff --git a/action.yml b/action.yml index c92e4fe..9bce3b6 100644 --- a/action.yml +++ b/action.yml @@ -39,14 +39,6 @@ inputs: description: 'Name of secret containing API key (OPENROUTER_API_KEY, OPENAI_API_KEY, or ANTHROPIC_API_KEY)' required: false default: 'OPENROUTER_API_KEY' - mcp-server-url: - description: 'HTTP URL of MCP server' - required: false - default: 'https://usable.dev/api/mcp' - mcp-token-secret: - description: 'Name of secret containing Usable API token' - required: false - default: 'USABLE_API_TOKEN' workspace-id: description: 'Usable workspace UUID (required - used to fetch MCP system prompt)' required: true @@ -291,8 +283,8 @@ runs: - name: Setup MCP Server shell: bash env: - MCP_URL: ${{ inputs.mcp-server-url }} - MCP_SECRET_NAME: ${{ inputs.mcp-token-secret }} + USABLE_URL: ${{ env.USABLE_URL || 'https://usable.dev' }} + USABLE_API_TOKEN: ${{ env.USABLE_API_TOKEN }} WORKSPACE_ID: ${{ inputs.workspace-id }} run: | ${{ github.action_path }}/scripts/setup-mcp.sh @@ -305,7 +297,8 @@ runs: WORKSPACE_ID: ${{ inputs.workspace-id }} CUSTOM_PROMPT_FILE: ${{ inputs.prompt-file }} MERGE_CUSTOM_PROMPT: ${{ inputs.merge-custom-prompt }} - MCP_SECRET_NAME: ${{ inputs.mcp-token-secret }} + USABLE_URL: ${{ env.USABLE_URL || 'https://usable.dev' }} + USABLE_API_TOKEN: ${{ env.USABLE_API_TOKEN }} ACTION_PATH: ${{ github.action_path }} run: | ${{ github.action_path }}/scripts/fetch-prompt.sh diff --git a/scripts/fetch-prompt.sh b/scripts/fetch-prompt.sh index b3f25f7..07a3f87 100755 --- a/scripts/fetch-prompt.sh +++ b/scripts/fetch-prompt.sh @@ -9,16 +9,21 @@ if ! command -v jq &> /dev/null; then exit 1 fi -# Get USABLE_API_TOKEN from secrets -USABLE_API_TOKEN="${!MCP_SECRET_NAME:-}" -if [ -z "$USABLE_API_TOKEN" ]; then +# Get USABLE_API_TOKEN from environment +# The token should be passed directly as USABLE_API_TOKEN env var +if [ -z "${USABLE_API_TOKEN:-}" ]; then echo "::warning::USABLE_API_TOKEN not found. Skipping MCP system prompt fetching." HAS_API_TOKEN=false else HAS_API_TOKEN=true + echo "โœ… USABLE_API_TOKEN found" fi -USABLE_API_BASE="https://usable.dev/api" +# Get Usable URL from environment (default to usable.dev) +USABLE_URL="${USABLE_URL:-https://usable.dev}" +USABLE_API_BASE="${USABLE_URL}/api" +echo "Usable API Base: $USABLE_API_BASE" + HARDCODED_SYSTEM_PROMPT="${ACTION_PATH}/system-prompt.md" MCP_SYSTEM_PROMPT_FILE="/tmp/mcp-system-prompt.md" USER_PROMPT_FILE="/tmp/user-prompt.md" @@ -90,20 +95,15 @@ fetch_mcp_system_prompt() { return 1 fi - # The API might return JSON with a content field, or plain text - # Try to parse as JSON first using jq - # Note: jq is pre-installed on GitHub Actions runners + # The API returns JSON with a systemPrompt field + # Extract it using jq local content - content=$(echo "$body" | jq -r '.content // .prompt // empty' 2>/dev/null) - - # If JSON parsing returned empty, use the body as-is (assuming plain text) - if [ -z "$content" ]; then - content="$body" - fi + content=$(echo "$body" | jq -r '.systemPrompt // empty' 2>/dev/null) # Verify content is not empty after parsing if [ -z "$content" ]; then echo "::warning::MCP system prompt content is empty after parsing." + echo "::debug::Response body: $body" return 1 fi @@ -169,15 +169,15 @@ main() { fi else # Static prompt file - if [ -n "$CUSTOM_PROMPT_FILE" ] && [ -f "$CUSTOM_PROMPT_FILE" ]; then + if [ -n "${CUSTOM_PROMPT_FILE:-}" ] && [ -f "$CUSTOM_PROMPT_FILE" ]; then echo "Using static prompt file: $CUSTOM_PROMPT_FILE" cp "$CUSTOM_PROMPT_FILE" "$USER_PROMPT_FILE" has_user_prompt=true echo "โœ… Static prompt loaded" echo "Size: $(wc -c < "$USER_PROMPT_FILE") bytes" else - echo "::error::No user prompt file provided or file not found" - exit 1 + echo "โ„น๏ธ No custom user prompt file specified - using only hardcoded system + MCP system prompts" + has_user_prompt=false fi fi diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index 97721af..111deaa 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -3,29 +3,19 @@ set -euo pipefail echo "::group::Setting up MCP Server Integration" -# Get the MCP token from environment -# In GitHub Actions: MCP_SECRET_NAME points to the env var name (e.g., USABLE_API_TOKEN) -# In local testing: Just use USABLE_API_TOKEN directly -if [ -n "${MCP_SECRET_NAME:-}" ]; then - # GitHub Actions mode: indirect reference - MCP_TOKEN="${!MCP_SECRET_NAME:-}" -else - # Local testing mode: direct token - MCP_TOKEN="${USABLE_API_TOKEN:-}" -fi - -if [ -z "$MCP_TOKEN" ]; then - echo "::error::MCP token not found" - echo "For GitHub Actions: Ensure env.${MCP_SECRET_NAME:-USABLE_API_TOKEN} is set" - echo "For local testing: Export USABLE_API_TOKEN environment variable" +# Get the Usable API token from environment +if [ -z "${USABLE_API_TOKEN:-}" ]; then + echo "::error::USABLE_API_TOKEN not found" + echo "Please set USABLE_API_TOKEN environment variable or secret" exit 1 fi -# Validate MCP URL -if [ -z "$MCP_URL" ]; then - echo "::error::MCP_URL is required when MCP is enabled" - exit 1 -fi +# Get Usable URL from environment (default to usable.dev) +USABLE_URL="${USABLE_URL:-https://usable.dev}" +MCP_ENDPOINT="${USABLE_URL}/api/mcp" + +echo "Usable URL: $USABLE_URL" +echo "MCP Endpoint: $MCP_ENDPOINT" # Remove any existing usable MCP server first (to avoid duplicates) forge mcp remove usable 2>/dev/null || echo " No existing usable server to remove" @@ -33,15 +23,15 @@ forge mcp remove usable 2>/dev/null || echo " No existing usable server to remo # Add Usable MCP server using HTTP transport with Authorization header # ForgeCode requires JSON configuration to pass HTTP headers echo "Adding Usable MCP server to ForgeCode..." -echo " URL: $MCP_URL" +echo " Endpoint: $MCP_ENDPOINT" echo " Auth: Bearer token (configured)" # Use forge mcp add-json to configure HTTP transport with Authorization header -forge mcp add-json "usable" "{\"url\":\"$MCP_URL\",\"headers\":{\"Authorization\":\"Bearer $MCP_TOKEN\"}}" 2>&1 +forge mcp add-json "usable" "{\"url\":\"$MCP_ENDPOINT\",\"headers\":{\"Authorization\":\"Bearer $USABLE_API_TOKEN\"}}" 2>&1 echo "โœ… MCP server configured for ForgeCode" echo " Transport: HTTP with Authorization header" -echo " Base URL: $MCP_URL" +echo " Endpoint: $MCP_ENDPOINT" # List configured servers for verification echo " Configured MCP servers:" diff --git a/test-local.sh b/test-local.sh index 2cae3db..e46409b 100755 --- a/test-local.sh +++ b/test-local.sh @@ -37,7 +37,7 @@ fi export PROVIDER="${PROVIDER:-auto}" export MODEL="${MODEL:-anthropic/claude-haiku-4.5}" export WORKSPACE_ID="${WORKSPACE_ID:-60c10ca2-4115-4c1a-b6d7-04ac39fd3938}" # Flowcore workspace -export MCP_SERVER_URL="${MCP_SERVER_URL:-https://usable.dev/api/mcp}" +export USABLE_URL="${USABLE_URL:-https://usable.dev}" # Git configuration (you can override these) export BASE_BRANCH="${BASE_BRANCH:-main}" @@ -53,6 +53,9 @@ export PR_LABELS="${PR_LABELS:-test,local}" # Other settings export USE_DYNAMIC_PROMPTS="${USE_DYNAMIC_PROMPTS:-false}" +# Don't set CUSTOM_PROMPT_FILE by default - let fetch-prompt.sh use only hardcoded + MCP prompts +# Users can set this if they want to test with a custom prompt file +export CUSTOM_PROMPT_FILE="${CUSTOM_PROMPT_FILE:-}" export PROMPT_FILE="${PROMPT_FILE:-./system-prompt.md}" export MAX_RETRIES="${MAX_RETRIES:-2}" export ALLOW_WEB_FETCH="${ALLOW_WEB_FETCH:-false}" @@ -112,17 +115,17 @@ echo "" # Run MCP setup echo "๐Ÿ”— Setting up MCP" echo "================" -# For local testing, MCP_URL is already set and USABLE_API_TOKEN is in the environment -MCP_URL="${MCP_SERVER_URL}" ./scripts/setup-mcp.sh +# For local testing, USABLE_URL and USABLE_API_TOKEN are in the environment +./scripts/setup-mcp.sh echo "" -# Check if we should fetch dynamic prompts -if [ "$USE_DYNAMIC_PROMPTS" = "true" ]; then - echo "๐Ÿ“ฅ Fetching Dynamic Prompts" - echo "==========================" - ./scripts/fetch-prompt.sh - echo "" -fi +# Check if we should fetch dynamic prompts or just prepare prompts +echo "๐Ÿ“ฅ Fetching/Preparing Prompts" +echo "=============================" +# Set ACTION_PATH for local testing (points to repo root where system-prompt.md is) +export ACTION_PATH="$(pwd)" +./scripts/fetch-prompt.sh +echo "" # Run validation echo "๐Ÿš€ Running Validation" From 7418553be82dd631be78d4cd22816ac178fbe4f0 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Wed, 22 Oct 2025 23:49:31 +0100 Subject: [PATCH 07/41] fix(validate.sh): correct variable assignment for status extraction - Updated the status variable assignment in the parse_results function to remove unnecessary local declaration, enhancing script clarity and functionality. --- scripts/validate.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/validate.sh b/scripts/validate.sh index 63b02af..6ad7d27 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -381,9 +381,10 @@ parse_results() { local validation_status local validation_passed local critical_issues + local status # Get status from JSON - local status=$(jq -r '.validationOutcome.status' "$json_file") + status=$(jq -r '.validationOutcome.status' "$json_file") if [ "$status" = "PASS" ]; then validation_status="passed" From cb2e863b194b33a94fb8e1531a9f8e32d6b6dd20 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Wed, 22 Oct 2025 23:53:19 +0100 Subject: [PATCH 08/41] feat(workflows): add MCP connection verification steps to validation jobs - Introduced a new step in the validation jobs to verify the Usable MCP connection by listing workspaces. - Added a report section for MCP connection status and workspace availability in the PR validation report. - Enhanced clarity and reliability of the validation process by ensuring the MCP connection is tested before proceeding with other tasks. --- .github/workflows/test.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e345718..56ccc7e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -238,6 +238,18 @@ jobs: ## Your Task Validate this PR for the usable-pr-validator GitHub Action repository. + ### Step 0: Verify MCP Connection + **Test the Usable MCP connection by listing workspaces:** + + Use the `list-workspaces` tool to verify: + - MCP server connection is working + - Authentication is successful + - You have access to the expected workspace(s) + + Expected workspace: "Flowcore" (ID: 60c10ca2-4115-4c1a-b6d7-04ac39fd3938) + + If the MCP connection fails, report this as a CRITICAL VIOLATION. + ### Step 1: Get PR Changes ```bash # Compare base ref (branch or tag) with HEAD @@ -279,6 +291,11 @@ jobs: # PR Validation Report + ## MCP Connection Test + - **Status**: Connected โœ… or Failed โŒ + - **Workspaces Found**: [count] + - **Target Workspace**: Found/Not Found + ## Summary [Brief overview of the changes and validation results] @@ -372,6 +389,9 @@ jobs: ## Your Task Validate documentation changes in this PR. + ### Step 0: Verify MCP Connection + Use the `list-workspaces` tool to verify the Usable MCP connection is working. + ### Get Changes ```bash # Compare base ref (branch or tag) with HEAD for markdown files @@ -399,6 +419,9 @@ jobs: # PR Validation Report + ## MCP Connection Test + - **Status**: Connected โœ… or Failed โŒ + ## Summary [Overview of documentation changes] @@ -489,6 +512,9 @@ jobs: ## Your Task Test the comment revalidation feature. + ### Step 0: Verify MCP Connection + Use the `list-workspaces` tool to verify the Usable MCP connection is working. + ### Step 1: Verify Override Comment Check if there is an override/clarification comment in the PR context above (marked with ๐Ÿ”„). @@ -515,6 +541,9 @@ jobs: # PR Validation Report + ## MCP Connection Test + - **Status**: Connected โœ… or Failed โŒ + ## Summary [Mention this is a comment revalidation test] [Note if override comment was detected] From c54f7f750449d64ba350c0f5da974e2d2c5541bb Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 00:00:23 +0100 Subject: [PATCH 09/41] feat(workflows): enhance MCP connection verification steps in validation jobs - Updated the validation jobs to emphasize the requirement of testing the Usable MCP connection before proceeding with other tasks. - Added detailed instructions for using the `mcp_usable_list-workspaces` tool, including expected outputs and critical validation criteria. - Improved the PR validation report format to include connection verification results, ensuring clarity on workspace accessibility and connection status. --- .github/workflows/test.yml | 58 +++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 56ccc7e..04b19d9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -238,17 +238,20 @@ jobs: ## Your Task Validate this PR for the usable-pr-validator GitHub Action repository. - ### Step 0: Verify MCP Connection - **Test the Usable MCP connection by listing workspaces:** + ### Step 0: Verify MCP Connection (REQUIRED - CRITICAL) + **YOU MUST TEST THE USABLE MCP CONNECTION FIRST:** - Use the `list-workspaces` tool to verify: - - MCP server connection is working - - Authentication is successful - - You have access to the expected workspace(s) + 1. Call the `mcp_usable_list-workspaces` tool with `{"includeArchived": false}` + 2. In your report, include the EXACT output showing: + - Total number of workspaces found + - Name of each workspace + - Whether the expected workspace "Flowcore" (ID: 60c10ca2-4115-4c1a-b6d7-04ac39fd3938) was found - Expected workspace: "Flowcore" (ID: 60c10ca2-4115-4c1a-b6d7-04ac39fd3938) + **CRITICAL**: If you do not call this tool and show the results, your report will be considered INVALID. - If the MCP connection fails, report this as a CRITICAL VIOLATION. + Expected result: Should find at least the "Flowcore" workspace. + + If the MCP connection fails, report this as a CRITICAL VIOLATION with the exact error message. ### Step 1: Get PR Changes ```bash @@ -291,10 +294,21 @@ jobs: # PR Validation Report - ## MCP Connection Test + ## MCP Connection Verification โœ… (REQUIRED) + + **MCP Server**: https://usable.dev/api/mcp + + ### Connection Test Results: - **Status**: Connected โœ… or Failed โŒ - - **Workspaces Found**: [count] - - **Target Workspace**: Found/Not Found + - **Tool Called**: `mcp_usable_list-workspaces` + - **Total Workspaces Found**: [exact number] + - **Workspaces**: + - [Workspace 1 name] (ID: [uuid]) + - [Workspace 2 name] (ID: [uuid]) + - ... + - **Target Workspace Found**: โœ… Flowcore (60c10ca2-4115-4c1a-b6d7-04ac39fd3938) or โŒ Not Found + + **Verification Stamp**: ๐Ÿ” MCP connection verified on [timestamp] ## Summary [Brief overview of the changes and validation results] @@ -389,8 +403,11 @@ jobs: ## Your Task Validate documentation changes in this PR. - ### Step 0: Verify MCP Connection - Use the `list-workspaces` tool to verify the Usable MCP connection is working. + ### Step 0: Verify MCP Connection (REQUIRED) + **YOU MUST call `mcp_usable_list-workspaces` tool first** and show the results in your report with: + - Total workspaces found + - List of workspace names and IDs + - Confirmation that "Flowcore" workspace was found ### Get Changes ```bash @@ -419,8 +436,11 @@ jobs: # PR Validation Report - ## MCP Connection Test + ## MCP Connection Verification โœ… - **Status**: Connected โœ… or Failed โŒ + - **Workspaces Found**: [count] + - **Target Workspace**: โœ… Flowcore found or โŒ Not Found + - **Verification Stamp**: ๐Ÿ” MCP verified ## Summary [Overview of documentation changes] @@ -512,8 +532,10 @@ jobs: ## Your Task Test the comment revalidation feature. - ### Step 0: Verify MCP Connection - Use the `list-workspaces` tool to verify the Usable MCP connection is working. + ### Step 0: Verify MCP Connection (REQUIRED) + **YOU MUST call `mcp_usable_list-workspaces` tool first** and show the results in your report with: + - Total workspaces found + - Verification that "Flowcore" workspace is accessible ### Step 1: Verify Override Comment @@ -541,8 +563,10 @@ jobs: # PR Validation Report - ## MCP Connection Test + ## MCP Connection Verification โœ… - **Status**: Connected โœ… or Failed โŒ + - **Workspaces Found**: [count] + - **Verification Stamp**: ๐Ÿ” MCP verified ## Summary [Mention this is a comment revalidation test] From 053ee6548e0e38b89b56aa6e47b456cdf762eced Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 00:04:58 +0100 Subject: [PATCH 10/41] fix(setup-mcp.sh, action.yml): remove MCP config with authentication token after use - Updated the setup-mcp.sh script to create a .mcp.json file for MCP server configuration with authentication details. - Added a cleanup step in action.yml to remove the .mcp.json file after the process to enhance security by eliminating sensitive data. --- action.yml | 1 + scripts/setup-mcp.sh | 43 +++++++++++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/action.yml b/action.yml index 9bce3b6..ac7db94 100644 --- a/action.yml +++ b/action.yml @@ -410,6 +410,7 @@ runs: # Remove temporary files containing sensitive data rm -f /tmp/forge-config.yaml rm -f /tmp/validation-*.md + rm -f .mcp.json # Remove MCP config with authentication token echo "โœ… Temporary files cleaned up" - name: Fail on Critical Violations diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index 111deaa..3b0f0a2 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -17,24 +17,35 @@ MCP_ENDPOINT="${USABLE_URL}/api/mcp" echo "Usable URL: $USABLE_URL" echo "MCP Endpoint: $MCP_ENDPOINT" -# Remove any existing usable MCP server first (to avoid duplicates) -forge mcp remove usable 2>/dev/null || echo " No existing usable server to remove" - -# Add Usable MCP server using HTTP transport with Authorization header -# ForgeCode requires JSON configuration to pass HTTP headers -echo "Adding Usable MCP server to ForgeCode..." +# Write .mcp.json file in the working directory with authentication +# ForgeCode loads MCP servers from .mcp.json in the current directory +echo "Writing .mcp.json configuration..." + +# Create the .mcp.json file with the Usable MCP server configuration +cat > .mcp.json <&1 - -echo "โœ… MCP server configured for ForgeCode" -echo " Transport: HTTP with Authorization header" -echo " Endpoint: $MCP_ENDPOINT" - -# List configured servers for verification -echo " Configured MCP servers:" -forge mcp list 2>&1 | sed 's/^/ /' || echo " (Unable to list servers)" +# Verify the configuration +if [ -f ".mcp.json" ]; then + echo " Configuration file created successfully" + echo " MCP server 'usable' will be available to ForgeCode" +else + echo "::error::Failed to create .mcp.json" + exit 1 +fi echo "::endgroup::" From 0a157d61edff4e5473df3ebbcf52a21535ced06d Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 00:42:22 +0100 Subject: [PATCH 11/41] feat(testing): enhance MCP connection testing and documentation - Updated README.md to include detailed instructions for running full validation tests and MCP-only connection tests, improving user guidance. - Modified test-local.sh to support a new `--mcp-only` mode for faster MCP connection verification. - Added a new script, test-mcp-connection.sh, to perform comprehensive checks on the MCP connection and output results. - Enhanced GitHub Actions workflow to include a dedicated MCP connection test step, ensuring that the Usable MCP connection is verified during pull request validation. - Improved output messages and result handling for better clarity and usability in testing scenarios. --- .github/workflows/test.yml | 41 +++++++++++ README.md | 22 +++++- scripts/setup-mcp.sh | 17 ++++- scripts/test-mcp-connection.sh | 127 +++++++++++++++++++++++++++++++++ test-local.sh | 20 +++++- 5 files changed, 220 insertions(+), 7 deletions(-) create mode 100755 scripts/test-mcp-connection.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 04b19d9..dfbf749 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -194,10 +194,49 @@ jobs: echo "โœ… Cleanup steps verified" + test-mcp-connection: + name: Test MCP Connection + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - uses: actions/checkout@v4 + + - name: Install ForgeCode + run: | + echo "Installing ForgeCode CLI..." + npm install -g forgecode@latest + forge --version + + - name: Setup MCP Server + env: + USABLE_URL: ${{ vars.USABLE_URL || 'https://usable.dev' }} + USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} + WORKSPACE_ID: '60c10ca2-4115-4c1a-b6d7-04ac39fd3938' + run: | + ./scripts/setup-mcp.sh + + - name: Test MCP Connection + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: | + ./scripts/test-mcp-connection.sh + + - name: Upload MCP Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: mcp-test-results + path: | + /tmp/mcp-test-*.txt + /tmp/mcp-test-*.json + retention-days: 7 + if-no-files-found: warn + integration-test: name: Integration Test runs-on: ubuntu-latest if: github.event_name == 'pull_request' + needs: test-mcp-connection permissions: contents: read pull-requests: write @@ -363,6 +402,7 @@ jobs: name: Integration Test (Documentation) runs-on: ubuntu-latest if: github.event_name == 'pull_request' + needs: test-mcp-connection permissions: contents: read pull-requests: write @@ -492,6 +532,7 @@ jobs: name: Integration Test (Comment Revalidation) runs-on: ubuntu-latest if: github.event_name == 'pull_request' + needs: test-mcp-connection permissions: contents: read pull-requests: write diff --git a/README.md b/README.md index c791949..391fb46 100644 --- a/README.md +++ b/README.md @@ -132,11 +132,14 @@ export HEAD_BRANCH='feature-branch' # current branch # Make sure you're on the branch you want to test git checkout your-feature-branch -# Run the test +# Run the full validation test ./test-local.sh + +# OR test MCP connection only (faster, verifies Usable integration) +./test-local.sh --mcp-only ``` -The script will: +The full test will: 1. Verify all required environment variables are set 2. Install ForgeCode CLI if not already installed 3. Fetch the necessary git refs @@ -144,12 +147,27 @@ The script will: 5. Run the validation 6. Display the results and save artifacts to `/tmp/` +The MCP-only test will: +1. Verify environment variables (API keys) +2. Install ForgeCode CLI if needed +3. Configure the Usable MCP server +4. Test that MCP tools are available to the AI model +5. Verify workspace access and authentication +6. Display connection results + +**Tip**: Run `--mcp-only` first to verify your Usable integration is working before running a full validation. + ### Viewing Results +**Full validation:** - **Full output**: `/tmp/validation-full-output.md` - **Validation report**: `/tmp/validation-report.md` - **GitHub outputs**: `/tmp/github-output.txt` +**MCP test:** +- **MCP test output**: `/tmp/mcp-test-output.txt` +- **MCP test results**: `/tmp/mcp-test-result.json` + ## ๐Ÿ“– Configuration ### Inputs diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index 3b0f0a2..9af2ec5 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -22,13 +22,16 @@ echo "MCP Endpoint: $MCP_ENDPOINT" echo "Writing .mcp.json configuration..." # Create the .mcp.json file with the Usable MCP server configuration +# ForgeCode uses "url" and "headers" for HTTP-based MCP servers (no transport field needed) cat > .mcp.json < /dev/null; then + echo "" + echo " Configuration preview:" + jq '.mcpServers | to_entries[] | {name: .key, url: .value.url, hasAuth: (.value.headers.Authorization != null)}' .mcp.json 2>/dev/null | sed 's/^/ /' || echo " (Preview unavailable)" + fi else echo "::error::Failed to create .mcp.json" exit 1 diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh new file mode 100755 index 0000000..0d15e2a --- /dev/null +++ b/scripts/test-mcp-connection.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "::group::Testing MCP Connection" + +# Check if .mcp.json exists +if [ ! -f ".mcp.json" ]; then + echo "::error::.mcp.json file not found" + echo "MCP server configuration is missing" + exit 1 +fi + +echo "โœ… .mcp.json file found" +echo "" +echo "Configuration:" +# Show config without exposing token +jq '.mcpServers | to_entries[] | {name: .key, url: .value.url, hasAuth: (.value.headers.Authorization != null)}' .mcp.json + +echo "" +echo "Creating MCP test prompt..." + +# Create a simple test prompt that lists available tools +cat > /tmp/mcp-test-prompt.txt <<'EOF' +# MCP Connection Test + +Your task is to verify the MCP connection by listing your available tools. + +## Instructions + +1. List ALL available tools you have access to +2. Look for tools that start with `mcp_usable_` prefix +3. If you find MCP tools, call `mcp_usable_list-workspaces` with parameter `{"includeArchived": false}` +4. Report the results + +## Output Format + +Write your findings to `/tmp/mcp-test-result.json` in this format: + +```json +{ + "mcpToolsFound": true/false, + "toolCount": 0, + "mcpTools": ["list", "of", "mcp", "tools"], + "workspacesResult": { + "success": true/false, + "workspaces": [], + "error": "error message if failed" + } +} +``` + +IMPORTANT: You must write the JSON file even if no MCP tools are found. +EOF + +echo "Running ForgeCode with MCP test prompt..." +echo "" + +# Run ForgeCode with the test prompt +if forge -p "$(cat /tmp/mcp-test-prompt.txt)" 2>&1 | tee /tmp/mcp-test-output.txt; then + echo "" + echo "ForgeCode execution completed" +else + echo "" + echo "::warning::ForgeCode execution had errors" +fi + +echo "" +echo "::group::ForgeCode Output" +cat /tmp/mcp-test-output.txt || echo "No output captured" +echo "::endgroup::" + +# Check if result file was created +if [ -f "/tmp/mcp-test-result.json" ]; then + echo "" + echo "::group::MCP Test Results" + cat /tmp/mcp-test-result.json + echo "" + echo "::endgroup::" + + # Parse results + if jq -e '.mcpToolsFound == true' /tmp/mcp-test-result.json > /dev/null 2>&1; then + echo "โœ… MCP tools are available to the AI model" + + # Check workspace test + if jq -e '.workspacesResult.success == true' /tmp/mcp-test-result.json > /dev/null 2>&1; then + WORKSPACE_COUNT=$(jq -r '.workspacesResult.workspaces | length' /tmp/mcp-test-result.json) + echo "โœ… MCP connection successful - found $WORKSPACE_COUNT workspace(s)" + + # List workspaces + echo "" + echo "Workspaces:" + jq -r '.workspacesResult.workspaces[] | " - \(.name) (ID: \(.id))"' /tmp/mcp-test-result.json || echo " (Unable to parse workspace list)" + + echo "::endgroup::" + exit 0 + else + echo "โŒ MCP tools found but workspace query failed" + jq -r '.workspacesResult.error // "Unknown error"' /tmp/mcp-test-result.json + echo "::endgroup::" + exit 1 + fi + else + echo "โŒ MCP tools NOT available to the AI model" + echo "" + echo "Available tools:" + jq -r '.mcpTools[]' /tmp/mcp-test-result.json || echo " (No tools information)" + echo "::endgroup::" + exit 1 + fi +else + echo "" + echo "::warning::AI model did not create result file" + echo "This likely means the model couldn't understand the prompt or write the file" + echo "" + echo "Checking output for MCP tool mentions..." + + if grep -q "mcp_usable" /tmp/mcp-test-output.txt; then + echo "โœ… Found references to mcp_usable tools in output" + else + echo "โŒ No references to mcp_usable tools found" + echo "::error::MCP tools are NOT available to the AI model" + fi + + echo "::endgroup::" + exit 1 +fi + diff --git a/test-local.sh b/test-local.sh index e46409b..ca3fea2 100755 --- a/test-local.sh +++ b/test-local.sh @@ -4,8 +4,16 @@ set -euo pipefail # Local testing script for PR validation # This allows you to test the validator locally before pushing to GitHub -echo "๐Ÿงช Usable PR Validator - Local Test" -echo "====================================" +# Parse command line arguments +TEST_MODE="full" +if [ "${1:-}" = "--mcp-only" ]; then + TEST_MODE="mcp" + echo "๐Ÿ”— Usable PR Validator - MCP Connection Test Only" + echo "==================================================" +else + echo "๐Ÿงช Usable PR Validator - Local Test" + echo "====================================" +fi echo "" # Check required environment variables @@ -119,6 +127,14 @@ echo "================" ./scripts/setup-mcp.sh echo "" +# If MCP-only test mode, run the MCP connection test and exit +if [ "$TEST_MODE" = "mcp" ]; then + echo "๐Ÿงช Running MCP Connection Test" + echo "==============================" + ./scripts/test-mcp-connection.sh + exit $? +fi + # Check if we should fetch dynamic prompts or just prepare prompts echo "๐Ÿ“ฅ Fetching/Preparing Prompts" echo "=============================" From 1c2695cfba5cb69ccc2c782b7ae3d0c103d0b29f Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 00:44:56 +0100 Subject: [PATCH 12/41] feat(docs, setup-mcp.sh): enhance MCP integration documentation and configuration - Updated README.md to include comprehensive details on automatic MCP integration, including configuration steps and testing instructions. - Modified setup-mcp.sh to improve the .mcp.json file creation process, ensuring it uses the correct environment variables and provides clearer output messages regarding the MCP server setup. - Enhanced documentation to clarify the automatic configuration of the Usable MCP server and the tools available for AI model access during validation. --- README.md | 36 ++++++++++++++++++++++++++++++++++++ scripts/setup-mcp.sh | 14 ++++++-------- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 391fb46..9ed914f 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,42 @@ The MCP-only test will: > **Note**: You must set the `USABLE_API_TOKEN` secret. Usable MCP integration is required for this action. +### ๐Ÿ”— MCP Integration (Automatic) + +The action automatically configures the **Usable MCP (Model Context Protocol) server** to give the AI model access to your team's knowledge base during validation. + +**How it works:** + +1. The action uses the `@usabledev/mcp-server` package (stdio transport) +2. Runs as a subprocess via `npx @usabledev/mcp-server@latest server` +3. Authenticates using the `USABLE_API_TOKEN` environment variable +4. Provides the AI model with tools like: + - `mcp_usable_list-workspaces` - List accessible workspaces + - `mcp_usable_agentic-search-fragments` - Search knowledge base + - `mcp_usable_get-memory-fragment-content` - Read full fragments + - `mcp_usable_create-memory-fragment` - Document deviations + - And more... + +**Configuration:** + +- **No manual setup required** - the action configures everything automatically +- **Node.js required** - The MCP server requires Node.js (available in GitHub Actions by default) +- **Network access** - The MCP server connects to your Usable instance to fetch knowledge + +**Testing MCP Connection:** + +```bash +# Test locally with the --mcp-only flag +./test-local.sh --mcp-only +``` + +This will verify that: + +- โœ… MCP server starts successfully +- โœ… AI model can access MCP tools +- โœ… Workspace authentication works +- โœ… Knowledge base is accessible + ### ๐Ÿง  System Prompts (Automatic) The action **automatically** includes system prompts to ensure high-quality validations. The final prompt is assembled in this order: diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index 9af2ec5..44801a7 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -12,17 +12,15 @@ fi # Get Usable URL from environment (default to usable.dev) USABLE_URL="${USABLE_URL:-https://usable.dev}" -MCP_ENDPOINT="${USABLE_URL}/api/mcp" echo "Usable URL: $USABLE_URL" -echo "MCP Endpoint: $MCP_ENDPOINT" # Write .mcp.json file in the working directory with authentication # ForgeCode loads MCP servers from .mcp.json in the current directory echo "Writing .mcp.json configuration..." # Create the .mcp.json file with the Usable MCP server configuration -# ForgeCode uses "url" and "headers" for HTTP-based MCP servers (no transport field needed) +# Using stdio transport via @usabledev/mcp-server package cat > .mcp.json < .mcp.json < .mcp.json < /dev/null; then echo "" echo " Configuration preview:" - jq '.mcpServers | to_entries[] | {name: .key, url: .value.url, hasAuth: (.value.headers.Authorization != null)}' .mcp.json 2>/dev/null | sed 's/^/ /' || echo " (Preview unavailable)" + jq '.mcpServers | to_entries[] | {name: .key, command: .value.command, hasToken: (.value.env.USABLE_API_TOKEN != null)}' .mcp.json 2>/dev/null | sed 's/^/ /' || echo " (Preview unavailable)" fi else echo "::error::Failed to create .mcp.json" From c4ba39129e852d547f7b650bad1cb50e3d346a14 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 00:46:14 +0100 Subject: [PATCH 13/41] feat(scripts): enhance MCP connection testing script - Updated test-mcp-connection.sh to include provider configuration setup, improving the testing process. - Modified output to display command and token presence for MCP servers, enhancing clarity on server capabilities. - Added error handling for missing setup-provider.sh, ensuring robust execution of the connection test. --- scripts/test-mcp-connection.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index 0d15e2a..a793ed4 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -14,11 +14,25 @@ echo "โœ… .mcp.json file found" echo "" echo "Configuration:" # Show config without exposing token -jq '.mcpServers | to_entries[] | {name: .key, url: .value.url, hasAuth: (.value.headers.Authorization != null)}' .mcp.json +jq '.mcpServers | to_entries[] | {name: .key, command: .value.command, hasToken: (.value.env.USABLE_API_TOKEN != null)}' .mcp.json echo "" echo "Creating MCP test prompt..." +# Setup provider configuration first (same as validate.sh) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +echo "" +echo "Setting up provider configuration..." +if [ -f "$SCRIPT_DIR/setup-provider.sh" ]; then + source "$SCRIPT_DIR/setup-provider.sh" || { + echo "::error::Failed to setup provider" + exit 1 + } +else + echo "::warning::setup-provider.sh not found, skipping provider setup" +fi + +echo "" # Create a simple test prompt that lists available tools cat > /tmp/mcp-test-prompt.txt <<'EOF' # MCP Connection Test From 443040b32967123b19f7abe396e5e1b2eed78858 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 08:04:10 +0100 Subject: [PATCH 14/41] feat(workflows): add default model variable for MCP connection testing - Introduced a new environment variable `MODEL` in the GitHub Actions workflow for MCP connection testing, defaulting to 'anthropic/claude-haiku-4.5' if not specified. This enhancement allows for more flexible model configuration during testing. --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dfbf749..d391383 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -218,6 +218,7 @@ jobs: - name: Test MCP Connection env: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + MODEL: ${{ vars.MODEL || 'anthropic/claude-haiku-4.5' }} run: | ./scripts/test-mcp-connection.sh From fbea8545e2d2b95dcad0d85625b1bf1398f8024b Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 08:08:07 +0100 Subject: [PATCH 15/41] feat(scripts, workflows): enhance MCP connection testing with model configuration - Added a new environment variable `PROVIDER` to the GitHub Actions workflow for MCP connection testing, defaulting to 'auto' if not specified. - Updated the `test-mcp-connection.sh` script to configure the ForgeCode model based on the `MODEL` variable, preventing interactive prompts during execution. This improves the flexibility and automation of the testing process. --- .github/workflows/test.yml | 1 + scripts/test-mcp-connection.sh | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d391383..273643f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -218,6 +218,7 @@ jobs: - name: Test MCP Connection env: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + PROVIDER: ${{ vars.PROVIDER || 'auto' }} MODEL: ${{ vars.MODEL || 'anthropic/claude-haiku-4.5' }} run: | ./scripts/test-mcp-connection.sh diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index a793ed4..a46d198 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -32,6 +32,16 @@ else echo "::warning::setup-provider.sh not found, skipping provider setup" fi +echo "" + +# Set ForgeCode model configuration (prevents interactive prompt) +if [ -n "${MODEL:-}" ]; then + echo "Configuring ForgeCode model: ${MODEL}" + forge config set --model "${MODEL}" 2>&1 || echo "::warning::Could not set model config" +else + echo "::warning::MODEL not set, ForgeCode may prompt for model selection" +fi + echo "" # Create a simple test prompt that lists available tools cat > /tmp/mcp-test-prompt.txt <<'EOF' From cec0f43faab233e4f40e4a6fff8220a810199def Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 08:11:22 +0100 Subject: [PATCH 16/41] feat(workflows, scripts): enhance MCP server setup and verification in testing - Added Node.js setup step in the GitHub Actions workflow to ensure compatibility with the latest features. - Included verification steps for the MCP server package and configuration in the workflow, improving reliability during testing. - Updated the test-mcp-connection.sh script to manually start the MCP server and verify its functionality, enhancing the robustness of the connection tests. --- .github/workflows/test.yml | 24 ++++++++++++++++++++++++ scripts/test-mcp-connection.sh | 15 +++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 273643f..8afb13f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -201,11 +201,22 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + - name: Install ForgeCode run: | echo "Installing ForgeCode CLI..." npm install -g forgecode@latest forge --version + + echo "" + echo "Node.js version:" + node --version + echo "npm version:" + npm --version - name: Setup MCP Server env: @@ -215,6 +226,19 @@ jobs: run: | ./scripts/setup-mcp.sh + - name: Verify MCP Server Package + run: | + echo "Testing if @usabledev/mcp-server can be executed..." + npx --yes @usabledev/mcp-server@latest --version || echo "::warning::MCP server version check failed" + + echo "" + echo "Checking .mcp.json configuration..." + cat .mcp.json + + echo "" + echo "ForgeCode MCP servers:" + forge mcp list + - name: Test MCP Connection env: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index a46d198..b76e9b5 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -76,6 +76,21 @@ Write your findings to `/tmp/mcp-test-result.json` in this format: IMPORTANT: You must write the JSON file even if no MCP tools are found. EOF +echo "Testing MCP server startup..." +echo "Attempting to start MCP server manually to verify it works..." + +# Test if the MCP server can start +timeout 5s npx --yes @usabledev/mcp-server@latest server 2>&1 | head -20 & +MCP_PID=$! +sleep 2 +if ps -p $MCP_PID > /dev/null 2>&1; then + echo "โœ… MCP server process started successfully (PID: $MCP_PID)" + kill $MCP_PID 2>/dev/null || true +else + echo "::warning::MCP server process failed to start or exited quickly" +fi + +echo "" echo "Running ForgeCode with MCP test prompt..." echo "" From 774205c7bc7ee2919164ab497cf7c5d3f1db4253 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 08:14:13 +0100 Subject: [PATCH 17/41] feat(workflows, scripts): add Usable API token and URL for enhanced MCP server testing - Introduced new environment variables `USABLE_API_TOKEN` and `USABLE_URL` in the GitHub Actions workflow to facilitate better MCP server connection testing. - Updated the `test-mcp-connection.sh` script to display the first 20 characters of the `USABLE_API_TOKEN` for verification and improved the manual start process with explicit environment variables. - Enhanced output messages to clarify the testing process and potential issues with environment variable passing. --- .github/workflows/test.yml | 2 ++ scripts/test-mcp-connection.sh | 15 +++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8afb13f..187c362 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -242,6 +242,8 @@ jobs: - name: Test MCP Connection env: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + USABLE_API_TOKEN: ${{ secrets.USABLE_API_TOKEN }} + USABLE_URL: ${{ vars.USABLE_URL || 'https://usable.dev' }} PROVIDER: ${{ vars.PROVIDER || 'auto' }} MODEL: ${{ vars.MODEL || 'anthropic/claude-haiku-4.5' }} run: | diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index b76e9b5..3a9235b 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -77,12 +77,15 @@ IMPORTANT: You must write the JSON file even if no MCP tools are found. EOF echo "Testing MCP server startup..." -echo "Attempting to start MCP server manually to verify it works..." +echo "Current USABLE_API_TOKEN in shell: ${USABLE_API_TOKEN:0:20}..." # Show first 20 chars only -# Test if the MCP server can start -timeout 5s npx --yes @usabledev/mcp-server@latest server 2>&1 | head -20 & +echo "" +echo "Attempting to start MCP server manually with explicit env vars..." + +# Test if the MCP server can start WITH explicit environment variables +USABLE_API_TOKEN="$USABLE_API_TOKEN" USABLE_BASE_URL="$USABLE_URL" timeout 5s npx --yes @usabledev/mcp-server@latest server 2>&1 | head -30 & MCP_PID=$! -sleep 2 +sleep 3 if ps -p $MCP_PID > /dev/null 2>&1; then echo "โœ… MCP server process started successfully (PID: $MCP_PID)" kill $MCP_PID 2>/dev/null || true @@ -90,6 +93,10 @@ else echo "::warning::MCP server process failed to start or exited quickly" fi +echo "" +echo "Note: If manual start works but ForgeCode doesn't see MCP tools," +echo "it means ForgeCode isn't properly passing env vars from .mcp.json" + echo "" echo "Running ForgeCode with MCP test prompt..." echo "" From a90a01633ce1b3c35d1b06fe5b5a88f2e9c0a2d7 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 08:17:23 +0100 Subject: [PATCH 18/41] feat(scripts): export environment variables for MCP server testing - Updated the `test-mcp-connection.sh` script to export `USABLE_API_TOKEN` and `USABLE_BASE_URL` as global environment variables for improved MCP server connection testing. - Enhanced output messages to confirm successful export of environment variables, aiding in troubleshooting and clarity during the testing process. --- scripts/test-mcp-connection.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index 3a9235b..746a14e 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -97,6 +97,14 @@ echo "" echo "Note: If manual start works but ForgeCode doesn't see MCP tools," echo "it means ForgeCode isn't properly passing env vars from .mcp.json" +echo "" +echo "Exporting environment variables for MCP server..." +# ForgeCode doesn't pass env vars from .mcp.json, so we export them globally +export USABLE_API_TOKEN="${USABLE_API_TOKEN}" +export USABLE_BASE_URL="${USABLE_URL}" +echo "โœ… USABLE_API_TOKEN exported (${USABLE_API_TOKEN:0:20}...)" +echo "โœ… USABLE_BASE_URL exported ($USABLE_BASE_URL)" + echo "" echo "Running ForgeCode with MCP test prompt..." echo "" From 98b8198a72d38bfa5c754a4075ad33f2531caa90 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 08:19:18 +0100 Subject: [PATCH 19/41] feat(scripts): improve MCP connection testing with process checks - Enhanced the `test-mcp-connection.sh` script to check for MCP server processes before and during the execution of ForgeCode, providing better visibility into the server's status. - Added output messages to indicate whether MCP server processes are found, improving troubleshooting capabilities during testing. - Modified the execution of ForgeCode to run in the background, allowing for monitoring of the MCP server subprocesses while the test is running. --- scripts/test-mcp-connection.sh | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index 746a14e..76e5a41 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -105,12 +105,26 @@ export USABLE_BASE_URL="${USABLE_URL}" echo "โœ… USABLE_API_TOKEN exported (${USABLE_API_TOKEN:0:20}...)" echo "โœ… USABLE_BASE_URL exported ($USABLE_BASE_URL)" +echo "" +echo "Checking for MCP server processes before ForgeCode starts..." +ps aux | grep -i "mcp-server\|@usabledev" | grep -v grep || echo "No MCP server processes found" + echo "" echo "Running ForgeCode with MCP test prompt..." echo "" -# Run ForgeCode with the test prompt -if forge -p "$(cat /tmp/mcp-test-prompt.txt)" 2>&1 | tee /tmp/mcp-test-output.txt; then +# Run ForgeCode with the test prompt in background to monitor +forge -p "$(cat /tmp/mcp-test-prompt.txt)" 2>&1 | tee /tmp/mcp-test-output.txt & +FORGE_PID=$! + +# Wait a bit and check if MCP server subprocess was spawned +sleep 2 +echo "" +echo "Checking for MCP server processes while ForgeCode is running..." +ps aux | grep -i "mcp-server\|@usabledev" | grep -v grep || echo "โš ๏ธ No MCP server subprocess found!" + +# Wait for ForgeCode to complete +if wait $FORGE_PID; then echo "" echo "ForgeCode execution completed" else From 1177ceffae4b599c4d719ac25e0f2d69d92346d0 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 08:22:26 +0100 Subject: [PATCH 20/41] feat(scripts): enhance MCP server setup with global installation - Added a step in `setup-mcp.sh` to install `@usabledev/mcp-server` globally for improved startup speed. - Updated output messages to reflect the new command for starting the MCP server, enhancing clarity in the setup process. - Ensured the .mcp.json configuration file is created with the correct server command, streamlining the user experience. --- scripts/setup-mcp.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index 44801a7..037f7c4 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -15,8 +15,15 @@ USABLE_URL="${USABLE_URL:-https://usable.dev}" echo "Usable URL: $USABLE_URL" +# Install @usabledev/mcp-server globally for faster startup +echo "" +echo "Installing @usabledev/mcp-server globally..." +npm install -g @usabledev/mcp-server@latest 2>&1 | grep -E "added|up to date|@usabledev" || echo " Installation output suppressed" +echo "โœ… MCP server package installed" + # Write .mcp.json file in the working directory with authentication # ForgeCode loads MCP servers from .mcp.json in the current directory +echo "" echo "Writing .mcp.json configuration..." # Create the .mcp.json file with the Usable MCP server configuration @@ -38,7 +45,7 @@ EOF echo "โœ… MCP server configured in .mcp.json" echo " Type: stdio (via @usabledev/mcp-server)" -echo " Command: npx @usabledev/mcp-server@latest server" +echo " Command: usable-mcp-server server (globally installed)" echo " Auth: Via USABLE_API_TOKEN environment variable" # Verify the configuration From 845e66069ae2183e19728181af4f8bcdf93ab4ec Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 08:40:30 +0100 Subject: [PATCH 21/41] feat(scripts): update MCP server setup script for improved output and security - Modified `setup-mcp.sh` to change the command for starting the MCP server to use `npx @usabledev/mcp-server@latest`, ensuring users always run the latest version. - Enhanced the configuration preview to include command arguments and added a new output section to display the full `.mcp.json` content with sensitive tokens masked, improving security and clarity during setup. --- scripts/setup-mcp.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index 037f7c4..c254976 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -45,7 +45,7 @@ EOF echo "โœ… MCP server configured in .mcp.json" echo " Type: stdio (via @usabledev/mcp-server)" -echo " Command: usable-mcp-server server (globally installed)" +echo " Command: npx @usabledev/mcp-server@latest server" echo " Auth: Via USABLE_API_TOKEN environment variable" # Verify the configuration @@ -57,8 +57,12 @@ if [ -f ".mcp.json" ]; then if command -v jq &> /dev/null; then echo "" echo " Configuration preview:" - jq '.mcpServers | to_entries[] | {name: .key, command: .value.command, hasToken: (.value.env.USABLE_API_TOKEN != null)}' .mcp.json 2>/dev/null | sed 's/^/ /' || echo " (Preview unavailable)" + jq '.mcpServers | to_entries[] | {name: .key, command: .value.command, args: .value.args, hasToken: (.value.env.USABLE_API_TOKEN != null)}' .mcp.json 2>/dev/null | sed 's/^/ /' || echo " (Preview unavailable)" fi + + echo "" + echo " Full .mcp.json content (with token masked):" + cat .mcp.json | sed "s/$USABLE_API_TOKEN/***MASKED***/g" | sed 's/^/ /' || echo " (Could not read file)" else echo "::error::Failed to create .mcp.json" exit 1 From 8184e3193ad0ec47d1b44927fb2fd1f4e83fe1de Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 10:04:55 +0100 Subject: [PATCH 22/41] feat(workflows): enhance .mcp.json configuration checks in testing workflow - Updated the GitHub Actions workflow to improve checks for the .mcp.json configuration file. - Added detailed output for the existence and content of .mcp.json, including masked sensitive tokens, parsed configuration, executed commands, and environment variables. - Implemented error handling for missing .mcp.json file, enhancing robustness and clarity during the testing process. --- .github/workflows/test.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 187c362..2470a36 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -233,7 +233,27 @@ jobs: echo "" echo "Checking .mcp.json configuration..." - cat .mcp.json + echo "File exists: $([ -f .mcp.json ] && echo 'YES' || echo 'NO')" + + if [ -f .mcp.json ]; then + echo "" + echo "Full .mcp.json content (with token masked for security):" + cat .mcp.json | sed 's/mmesh_[a-zA-Z0-9_]*/mmesh_***MASKED***/g' + + echo "" + echo "Parsed configuration:" + jq '.' .mcp.json 2>&1 || echo "::warning::Failed to parse JSON" + + echo "" + echo "Command that will be executed:" + jq -r '.mcpServers.usable.command + " " + (.mcpServers.usable.args | join(" "))' .mcp.json 2>&1 || echo "::warning::Failed to extract command" + + echo "" + echo "Environment variables configured in .mcp.json:" + jq -r '.mcpServers.usable.env | keys[]' .mcp.json 2>&1 || echo "::warning::Failed to extract env vars" + else + echo "::error::.mcp.json file not found!" + fi echo "" echo "ForgeCode MCP servers:" From c8e48f441f0fd920b4029fec53f35fa4ad3e973e Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 10:06:04 +0100 Subject: [PATCH 23/41] fix(scripts): remove USABLE_BASE_URL from MCP server setup script - Updated `setup-mcp.sh` to remove the `USABLE_BASE_URL` environment variable from the configuration, streamlining the setup process and enhancing security by limiting exposed sensitive information. --- scripts/setup-mcp.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index c254976..1f50d2a 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -35,8 +35,7 @@ cat > .mcp.json < Date: Thu, 23 Oct 2025 10:08:58 +0100 Subject: [PATCH 24/41] fix(workflows, scripts): improve security and output handling in testing workflow - Updated the GitHub Actions workflow to partially mask sensitive tokens in the `.mcp.json` output for enhanced security. - Modified the `test-mcp-connection.sh` script to capture both stdout and stderr separately, providing clearer error reporting and better visibility into MCP initialization. - Added checks for errors in stderr during the MCP server process, improving troubleshooting capabilities. --- .github/workflows/test.yml | 4 ++-- scripts/test-mcp-connection.sh | 20 +++++++++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2470a36..fd3ace3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -237,8 +237,8 @@ jobs: if [ -f .mcp.json ]; then echo "" - echo "Full .mcp.json content (with token masked for security):" - cat .mcp.json | sed 's/mmesh_[a-zA-Z0-9_]*/mmesh_***MASKED***/g' + echo "Full .mcp.json content (with token partially masked for security):" + cat .mcp.json | sed -E 's/(mmesh_[a-zA-Z0-9_]{6})[a-zA-Z0-9_]*/\1***/g' echo "" echo "Parsed configuration:" diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index 76e5a41..d6c987a 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -112,17 +112,31 @@ ps aux | grep -i "mcp-server\|@usabledev" | grep -v grep || echo "No MCP server echo "" echo "Running ForgeCode with MCP test prompt..." echo "" +echo "Capturing both stdout and stderr to see MCP initialization..." -# Run ForgeCode with the test prompt in background to monitor -forge -p "$(cat /tmp/mcp-test-prompt.txt)" 2>&1 | tee /tmp/mcp-test-output.txt & +# Run ForgeCode with the test prompt, capturing all output including stderr +forge -p "$(cat /tmp/mcp-test-prompt.txt)" > /tmp/mcp-test-stdout.txt 2> /tmp/mcp-test-stderr.txt & FORGE_PID=$! -# Wait a bit and check if MCP server subprocess was spawned +# Wait a bit and check for MCP server subprocess and any errors sleep 2 echo "" echo "Checking for MCP server processes while ForgeCode is running..." ps aux | grep -i "mcp-server\|@usabledev" | grep -v grep || echo "โš ๏ธ No MCP server subprocess found!" +echo "" +echo "Checking for errors in stderr..." +if [ -s /tmp/mcp-test-stderr.txt ]; then + echo "::group::ForgeCode stderr (may contain MCP initialization errors)" + cat /tmp/mcp-test-stderr.txt + echo "::endgroup::" +else + echo "No errors in stderr" +fi + +# Combine stdout and stderr for full output +cat /tmp/mcp-test-stdout.txt /tmp/mcp-test-stderr.txt > /tmp/mcp-test-output.txt 2>&1 + # Wait for ForgeCode to complete if wait $FORGE_PID; then echo "" From 38369f89e70882c353d422be9d47af1d236ab617 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 10:10:28 +0100 Subject: [PATCH 25/41] fix(scripts): add newline at the end of test-mcp-connection.sh - Added a newline at the end of the `test-mcp-connection.sh` script to adhere to best practices for shell scripts, improving readability and preventing potential issues in execution. --- scripts/test-mcp-connection.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index d6c987a..1303294 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -207,3 +207,4 @@ else exit 1 fi + From 4ae037faa447d10b3f5b88a9a7a648fa06df7f98 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 10:17:16 +0100 Subject: [PATCH 26/41] feat(scripts): add MCP protocol testing script and enhance connection script output - Introduced `test-mcp-protocol.sh` to validate MCP server responses to protocol requests over stdio, improving testing capabilities. - Enhanced `test-mcp-connection.sh` to include current working directory and existence check for `.mcp.json`, aiding in troubleshooting and clarity during the testing process. --- scripts/test-mcp-connection.sh | 2 ++ scripts/test-mcp-protocol.sh | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100755 scripts/test-mcp-protocol.sh diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index 1303294..204d0ac 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -111,6 +111,8 @@ ps aux | grep -i "mcp-server\|@usabledev" | grep -v grep || echo "No MCP server echo "" echo "Running ForgeCode with MCP test prompt..." +echo "Current working directory: $(pwd)" +echo ".mcp.json exists in PWD: $([ -f .mcp.json ] && echo 'YES' || echo 'NO')" echo "" echo "Capturing both stdout and stderr to see MCP initialization..." diff --git a/scripts/test-mcp-protocol.sh b/scripts/test-mcp-protocol.sh new file mode 100755 index 0000000..2395088 --- /dev/null +++ b/scripts/test-mcp-protocol.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Test if the MCP server responds to MCP protocol requests over stdio + +echo "Testing MCP server stdio protocol..." +echo "" + +# Export environment variables +export USABLE_API_TOKEN="${USABLE_API_TOKEN}" +export USABLE_BASE_URL="${USABLE_URL:-https://usable.dev}" + +echo "Starting MCP server and sending initialize request..." +echo "" + +# Send an MCP initialize request via stdin +echo '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"test","version":"1.0"}}}' | \ + npx --yes @usabledev/mcp-server@latest server 2>&1 | head -50 + +echo "" +echo "If you see a valid JSON-RPC response above with server info, the MCP server works." +echo "If you see errors or no response, the MCP server has issues." + From fc637fc0632d1ed55b344901ab421a7d91d99da6 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 10:20:48 +0100 Subject: [PATCH 27/41] feat(scripts): enhance MCP connection testing with additional checks and debug logging - Added checks to verify if ForgeCode can see MCP servers in the configuration, improving visibility into server connectivity. - Implemented a longer wait time for MCP initialization to ensure processes are fully started before checks. - Enhanced error reporting by checking stderr for MCP initialization logs and capturing debug logs with RUST_LOG enabled, aiding in troubleshooting. - Improved output handling to display combined output and debug logs, providing clearer insights during testing. --- scripts/test-mcp-connection.sh | 38 ++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index 204d0ac..a2e6097 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -76,6 +76,11 @@ Write your findings to `/tmp/mcp-test-result.json` in this format: IMPORTANT: You must write the JSON file even if no MCP tools are found. EOF +echo "" +echo "Checking if ForgeCode can see MCP servers in config..." +forge mcp list 2>&1 || echo "โš ๏ธ forge mcp list command not available or failed" + +echo "" echo "Testing MCP server startup..." echo "Current USABLE_API_TOKEN in shell: ${USABLE_API_TOKEN:0:20}..." # Show first 20 chars only @@ -117,15 +122,25 @@ echo "" echo "Capturing both stdout and stderr to see MCP initialization..." # Run ForgeCode with the test prompt, capturing all output including stderr -forge -p "$(cat /tmp/mcp-test-prompt.txt)" > /tmp/mcp-test-stdout.txt 2> /tmp/mcp-test-stderr.txt & +# Also enable ForgeCode debug logging to see MCP initialization +RUST_LOG=debug forge -p "$(cat /tmp/mcp-test-prompt.txt)" > /tmp/mcp-test-stdout.txt 2> /tmp/mcp-test-stderr.txt & FORGE_PID=$! -# Wait a bit and check for MCP server subprocess and any errors -sleep 2 +# Wait a bit longer for MCP initialization and check for processes +sleep 5 echo "" echo "Checking for MCP server processes while ForgeCode is running..." ps aux | grep -i "mcp-server\|@usabledev" | grep -v grep || echo "โš ๏ธ No MCP server subprocess found!" +# Check if there are any MCP-related errors in system logs +echo "" +echo "Checking forge stderr for MCP initialization logs..." +if [ -f /tmp/mcp-test-stderr.txt ] && [ -s /tmp/mcp-test-stderr.txt ]; then + grep -i "mcp\|server\|stdio\|initialize" /tmp/mcp-test-stderr.txt || echo "No MCP-related logs in stderr" +else + echo "stderr file is empty or doesn't exist yet" +fi + echo "" echo "Checking for errors in stderr..." if [ -s /tmp/mcp-test-stderr.txt ]; then @@ -150,7 +165,22 @@ fi echo "" echo "::group::ForgeCode Output" -cat /tmp/mcp-test-output.txt || echo "No output captured" +if [ -f /tmp/mcp-test-output.txt ]; then + echo "Combined output (first 100 lines):" + head -100 /tmp/mcp-test-output.txt +else + echo "No output captured" +fi +echo "::endgroup::" + +echo "" +echo "::group::ForgeCode Debug Logs (stderr - with RUST_LOG=debug)" +if [ -f /tmp/mcp-test-stderr.txt ] && [ -s /tmp/mcp-test-stderr.txt ]; then + echo "Debug logs (first 300 lines - may show MCP initialization details):" + head -300 /tmp/mcp-test-stderr.txt +else + echo "No debug logs captured" +fi echo "::endgroup::" # Check if result file was created From c2728667e7008294608dfb9ce007fb493491336e Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 10:25:43 +0100 Subject: [PATCH 28/41] refactor(scripts): streamline MCP connection testing and enhance output handling - Removed manual MCP server startup checks to simplify the script and focus on ForgeCode's ability to detect MCP servers. - Added a check for available tools before running ForgeCode, improving the initial setup verification. - Implemented verbose output for ForgeCode execution, capturing detailed logs for better troubleshooting. - Updated log checks to search for MCP-related patterns in the output, enhancing error detection and reporting capabilities. --- scripts/test-mcp-connection.sh | 66 +++++++++++----------------------- 1 file changed, 21 insertions(+), 45 deletions(-) diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index a2e6097..5f95adb 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -81,26 +81,8 @@ echo "Checking if ForgeCode can see MCP servers in config..." forge mcp list 2>&1 || echo "โš ๏ธ forge mcp list command not available or failed" echo "" -echo "Testing MCP server startup..." -echo "Current USABLE_API_TOKEN in shell: ${USABLE_API_TOKEN:0:20}..." # Show first 20 chars only - -echo "" -echo "Attempting to start MCP server manually with explicit env vars..." - -# Test if the MCP server can start WITH explicit environment variables -USABLE_API_TOKEN="$USABLE_API_TOKEN" USABLE_BASE_URL="$USABLE_URL" timeout 5s npx --yes @usabledev/mcp-server@latest server 2>&1 | head -30 & -MCP_PID=$! -sleep 3 -if ps -p $MCP_PID > /dev/null 2>&1; then - echo "โœ… MCP server process started successfully (PID: $MCP_PID)" - kill $MCP_PID 2>/dev/null || true -else - echo "::warning::MCP server process failed to start or exited quickly" -fi - -echo "" -echo "Note: If manual start works but ForgeCode doesn't see MCP tools," -echo "it means ForgeCode isn't properly passing env vars from .mcp.json" +echo "Checking available tools before ForgeCode runs..." +forge show-tools 2>&1 | head -50 || echo "โš ๏ธ forge show-tools command failed" echo "" echo "Exporting environment variables for MCP server..." @@ -122,8 +104,9 @@ echo "" echo "Capturing both stdout and stderr to see MCP initialization..." # Run ForgeCode with the test prompt, capturing all output including stderr -# Also enable ForgeCode debug logging to see MCP initialization -RUST_LOG=debug forge -p "$(cat /tmp/mcp-test-prompt.txt)" > /tmp/mcp-test-stdout.txt 2> /tmp/mcp-test-stderr.txt & +# Use --verbose flag for detailed output +echo "Running ForgeCode with --verbose flag..." +forge --verbose -p "$(cat /tmp/mcp-test-prompt.txt)" 2>&1 | tee /tmp/mcp-test-output.txt & FORGE_PID=$! # Wait a bit longer for MCP initialization and check for processes @@ -132,28 +115,21 @@ echo "" echo "Checking for MCP server processes while ForgeCode is running..." ps aux | grep -i "mcp-server\|@usabledev" | grep -v grep || echo "โš ๏ธ No MCP server subprocess found!" -# Check if there are any MCP-related errors in system logs +# Check for any MCP-related patterns in the output echo "" -echo "Checking forge stderr for MCP initialization logs..." -if [ -f /tmp/mcp-test-stderr.txt ] && [ -s /tmp/mcp-test-stderr.txt ]; then - grep -i "mcp\|server\|stdio\|initialize" /tmp/mcp-test-stderr.txt || echo "No MCP-related logs in stderr" -else - echo "stderr file is empty or doesn't exist yet" -fi - -echo "" -echo "Checking for errors in stderr..." -if [ -s /tmp/mcp-test-stderr.txt ]; then - echo "::group::ForgeCode stderr (may contain MCP initialization errors)" - cat /tmp/mcp-test-stderr.txt - echo "::endgroup::" +echo "Checking for MCP-related logs in output..." +if [ -f /tmp/mcp-test-output.txt ] && [ -s /tmp/mcp-test-output.txt ]; then + if grep -qi "mcp\|server\|stdio\|initialize\|spawn\|subprocess" /tmp/mcp-test-output.txt; then + echo "::group::MCP-related logs found" + grep -i "mcp\|server\|stdio\|initialize\|spawn\|subprocess" /tmp/mcp-test-output.txt | head -50 + echo "::endgroup::" + else + echo "No MCP-related patterns found in output" + fi else - echo "No errors in stderr" + echo "Output file is empty or doesn't exist yet" fi -# Combine stdout and stderr for full output -cat /tmp/mcp-test-stdout.txt /tmp/mcp-test-stderr.txt > /tmp/mcp-test-output.txt 2>&1 - # Wait for ForgeCode to complete if wait $FORGE_PID; then echo "" @@ -174,12 +150,12 @@ fi echo "::endgroup::" echo "" -echo "::group::ForgeCode Debug Logs (stderr - with RUST_LOG=debug)" -if [ -f /tmp/mcp-test-stderr.txt ] && [ -s /tmp/mcp-test-stderr.txt ]; then - echo "Debug logs (first 300 lines - may show MCP initialization details):" - head -300 /tmp/mcp-test-stderr.txt +echo "::group::ForgeCode Verbose Output (with --verbose flag)" +if [ -f /tmp/mcp-test-output.txt ] && [ -s /tmp/mcp-test-output.txt ]; then + echo "Verbose output (first 300 lines - may show MCP initialization details):" + tail -n +2 /tmp/mcp-test-output.txt | head -300 # Skip first line (duplicate) else - echo "No debug logs captured" + echo "No verbose output captured" fi echo "::endgroup::" From 291c43d5b6f01cd7a28f01df9bdda7e81f5eb066 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 11:11:40 +0100 Subject: [PATCH 29/41] fix(scripts): :zap: switch MCP from stdio to HTTP transport for CI compatibility - Change from stdio subprocess to direct HTTP connection to Usable's MCP API - Avoids stdio buffering/sandboxing issues in GitHub Actions CI environments - Based on Perplexity research showing stdio transport limitations in CI - HTTP transport is more reliable across CI/CD runner boundaries --- scripts/setup-mcp.sh | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index 1f50d2a..2c284b6 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -15,27 +15,22 @@ USABLE_URL="${USABLE_URL:-https://usable.dev}" echo "Usable URL: $USABLE_URL" -# Install @usabledev/mcp-server globally for faster startup -echo "" -echo "Installing @usabledev/mcp-server globally..." -npm install -g @usabledev/mcp-server@latest 2>&1 | grep -E "added|up to date|@usabledev" || echo " Installation output suppressed" -echo "โœ… MCP server package installed" - -# Write .mcp.json file in the working directory with authentication +# Write .mcp.json file with HTTP transport (more reliable in CI than stdio) # ForgeCode loads MCP servers from .mcp.json in the current directory echo "" echo "Writing .mcp.json configuration..." -# Create the .mcp.json file with the Usable MCP server configuration -# Using stdio transport via @usabledev/mcp-server package +# Create the .mcp.json file with HTTP transport to Usable's MCP endpoint +# This avoids stdio buffering issues in GitHub Actions CI cat > .mcp.json < .mcp.json < /dev/null; then echo "" echo " Configuration preview:" - jq '.mcpServers | to_entries[] | {name: .key, command: .value.command, args: .value.args, hasToken: (.value.env.USABLE_API_TOKEN != null)}' .mcp.json 2>/dev/null | sed 's/^/ /' || echo " (Preview unavailable)" + jq '.mcpServers | to_entries[] | {name: .key, transport: .value.transport, url: .value.url, hasAuth: (.value.headers.Authorization != null)}' .mcp.json 2>/dev/null | sed 's/^/ /' || echo " (Preview unavailable)" fi echo "" From f470877f167566768909874b5a1c2c3d1100c618 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 11:38:48 +0100 Subject: [PATCH 30/41] fix(scripts): :wrench: use streaming HTTP format with env.AUTHORIZATION - Change from headers to env.AUTHORIZATION format for better compatibility - Matches ForgeCode's secure_server streaming HTTP example format - Should work with ForgeCode's HTTP streaming transport handler --- scripts/setup-mcp.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index 2c284b6..71cff75 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -15,22 +15,21 @@ USABLE_URL="${USABLE_URL:-https://usable.dev}" echo "Usable URL: $USABLE_URL" -# Write .mcp.json file with HTTP transport (more reliable in CI than stdio) +# Write .mcp.json file with streaming HTTP transport (more reliable in CI than stdio) # ForgeCode loads MCP servers from .mcp.json in the current directory echo "" echo "Writing .mcp.json configuration..." -# Create the .mcp.json file with HTTP transport to Usable's MCP endpoint +# Create the .mcp.json file with streaming HTTP transport to Usable's MCP endpoint # This avoids stdio buffering issues in GitHub Actions CI +# Using env.AUTHORIZATION format instead of headers for better compatibility cat > .mcp.json < .mcp.json < /dev/null; then echo "" echo " Configuration preview:" - jq '.mcpServers | to_entries[] | {name: .key, transport: .value.transport, url: .value.url, hasAuth: (.value.headers.Authorization != null)}' .mcp.json 2>/dev/null | sed 's/^/ /' || echo " (Preview unavailable)" + jq '.mcpServers | to_entries[] | {name: .key, url: .value.url, hasAuth: (.value.env.AUTHORIZATION != null)}' .mcp.json 2>/dev/null | sed 's/^/ /' || echo " (Preview unavailable)" fi echo "" From c9dc80c001170f1d57fefe80e8d375aa0b769ab4 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 11:46:57 +0100 Subject: [PATCH 31/41] refactor(scripts): :recycle: remove stdio-specific checks from MCP test - Remove checks for MCP server subprocesses (no subprocess with HTTP) - Remove environment variable exports (not needed for HTTP transport) - Update messages to reflect HTTP transport usage - Simplify test flow for HTTP-based MCP connection - Look for 'http' and 'connect' patterns instead of 'stdio' and 'spawn' --- scripts/test-mcp-connection.sh | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index 5f95adb..9e0f960 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -81,20 +81,9 @@ echo "Checking if ForgeCode can see MCP servers in config..." forge mcp list 2>&1 || echo "โš ๏ธ forge mcp list command not available or failed" echo "" -echo "Checking available tools before ForgeCode runs..." -forge show-tools 2>&1 | head -50 || echo "โš ๏ธ forge show-tools command failed" - -echo "" -echo "Exporting environment variables for MCP server..." -# ForgeCode doesn't pass env vars from .mcp.json, so we export them globally -export USABLE_API_TOKEN="${USABLE_API_TOKEN}" -export USABLE_BASE_URL="${USABLE_URL}" -echo "โœ… USABLE_API_TOKEN exported (${USABLE_API_TOKEN:0:20}...)" -echo "โœ… USABLE_BASE_URL exported ($USABLE_BASE_URL)" - -echo "" -echo "Checking for MCP server processes before ForgeCode starts..." -ps aux | grep -i "mcp-server\|@usabledev" | grep -v grep || echo "No MCP server processes found" +echo "Note: Using HTTP transport to ${USABLE_URL}/api/mcp" +echo " This avoids stdio subprocess issues in CI environments" +echo " Authorization is passed via env.AUTHORIZATION in .mcp.json" echo "" echo "Running ForgeCode with MCP test prompt..." @@ -109,19 +98,17 @@ echo "Running ForgeCode with --verbose flag..." forge --verbose -p "$(cat /tmp/mcp-test-prompt.txt)" 2>&1 | tee /tmp/mcp-test-output.txt & FORGE_PID=$! -# Wait a bit longer for MCP initialization and check for processes +# Wait for ForgeCode to initialize and connect to HTTP MCP server +echo "Waiting for ForgeCode to connect to HTTP MCP server..." sleep 5 -echo "" -echo "Checking for MCP server processes while ForgeCode is running..." -ps aux | grep -i "mcp-server\|@usabledev" | grep -v grep || echo "โš ๏ธ No MCP server subprocess found!" # Check for any MCP-related patterns in the output echo "" echo "Checking for MCP-related logs in output..." if [ -f /tmp/mcp-test-output.txt ] && [ -s /tmp/mcp-test-output.txt ]; then - if grep -qi "mcp\|server\|stdio\|initialize\|spawn\|subprocess" /tmp/mcp-test-output.txt; then + if grep -qi "mcp\|server\|http\|initialize\|connect" /tmp/mcp-test-output.txt; then echo "::group::MCP-related logs found" - grep -i "mcp\|server\|stdio\|initialize\|spawn\|subprocess" /tmp/mcp-test-output.txt | head -50 + grep -i "mcp\|server\|http\|initialize\|connect" /tmp/mcp-test-output.txt | head -50 echo "::endgroup::" else echo "No MCP-related patterns found in output" From 55481bec271afa614583d2c9af896a56b5d5cf30 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 12:02:53 +0100 Subject: [PATCH 32/41] feat(action): enhance PR validation process with dynamic prompt handling and improved output reporting - Added logic to determine the prompt file to use, supporting both dynamic and static prompts. - Configured ForgeCode model settings if specified, improving flexibility in validation. - Enhanced output reporting by capturing validation results and generating a markdown report from JSON, providing clearer insights into validation outcomes. - Updated the .mcp.json setup to utilize stdio transport via npx for better subprocess control and reliability in CI environments. --- action.yml | 74 +++++++++++++++++++++++++++++++++++++++++++- scripts/setup-mcp.sh | 25 ++++++++------- 2 files changed, 86 insertions(+), 13 deletions(-) diff --git a/action.yml b/action.yml index ac7db94..d213a83 100644 --- a/action.yml +++ b/action.yml @@ -325,7 +325,79 @@ runs: ALLOW_WEB_FETCH: ${{ inputs.allow-web-fetch }} GIT_PAGER: cat run: | - ${{ github.action_path }}/scripts/validate.sh + echo "::group::Running PR Validation" + + # Determine which prompt file to use + if [ -f "/tmp/dynamic-prompt.md" ]; then + ACTUAL_PROMPT="/tmp/dynamic-prompt.md" + echo "Using dynamic prompt from fetch-prompt.sh" + elif [ -n "$PROMPT_FILE" ] && [ -f "$PROMPT_FILE" ]; then + ACTUAL_PROMPT="$PROMPT_FILE" + echo "Using static prompt file: $PROMPT_FILE" + else + echo "::error::No valid prompt file found" + exit 1 + fi + + # Configure ForgeCode model if specified + if [ -n "$MODEL" ]; then + echo "Configuring ForgeCode model: $MODEL" + forge config set --model "$MODEL" || echo "Warning: Could not set model config" + fi + + # Run ForgeCode directly with MCP stdio transport + echo "๐Ÿค– Running ForgeCode CLI with MCP stdio transport" + echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" + echo "Provider: ${PROVIDER:-auto}" + echo "Model: $MODEL" + echo "Prompt: $ACTUAL_PROMPT" + echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" + + # Run forge and capture output + set +e + forge -p "$(cat "$ACTUAL_PROMPT")" 2>&1 | tee /tmp/validation-full-output.md + FORGE_EXIT=$? + set -e + + echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" + + if [ $FORGE_EXIT -ne 0 ]; then + echo "::error::ForgeCode failed with exit code: $FORGE_EXIT" + echo "validation_status=error" >> $GITHUB_OUTPUT + echo "validation_passed=false" >> $GITHUB_OUTPUT + echo "critical_issues=0" >> $GITHUB_OUTPUT + exit 1 + fi + + # Process the JSON report if it exists + if [ -f "/tmp/validation-report.json" ]; then + # Construct markdown report + ${{ github.action_path }}/scripts/construct-markdown.sh /tmp/validation-report.json /tmp/validation-report.md + + # Parse results + STATUS=$(jq -r '.validationOutcome.status' /tmp/validation-report.json) + CRITICAL=$(jq -r '.validationOutcome.criticalIssuesCount' /tmp/validation-report.json) + + if [ "$STATUS" = "PASS" ]; then + echo "validation_status=passed" >> $GITHUB_OUTPUT + echo "validation_passed=true" >> $GITHUB_OUTPUT + else + echo "validation_status=failed" >> $GITHUB_OUTPUT + echo "validation_passed=false" >> $GITHUB_OUTPUT + fi + + echo "critical_issues=${CRITICAL:-0}" >> $GITHUB_OUTPUT + + echo "โœ… Validation completed: $STATUS (Critical: ${CRITICAL:-0})" + else + echo "::warning::No JSON report generated" + cp /tmp/validation-full-output.md /tmp/validation-report.md || true + echo "validation_status=completed" >> $GITHUB_OUTPUT + echo "validation_passed=true" >> $GITHUB_OUTPUT + echo "critical_issues=0" >> $GITHUB_OUTPUT + fi + + echo "::endgroup::" - name: Post PR Comment if: inputs.comment-mode != 'none' && always() diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index 71cff75..ce5f588 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -15,21 +15,22 @@ USABLE_URL="${USABLE_URL:-https://usable.dev}" echo "Usable URL: $USABLE_URL" -# Write .mcp.json file with streaming HTTP transport (more reliable in CI than stdio) +# Write .mcp.json file with stdio transport using npx # ForgeCode loads MCP servers from .mcp.json in the current directory echo "" echo "Writing .mcp.json configuration..." -# Create the .mcp.json file with streaming HTTP transport to Usable's MCP endpoint -# This avoids stdio buffering issues in GitHub Actions CI -# Using env.AUTHORIZATION format instead of headers for better compatibility +# Create the .mcp.json file with stdio transport using npx +# This uses the Usable MCP server package via npx for better subprocess control cat > .mcp.json < .mcp.json < /dev/null; then echo "" echo " Configuration preview:" - jq '.mcpServers | to_entries[] | {name: .key, url: .value.url, hasAuth: (.value.env.AUTHORIZATION != null)}' .mcp.json 2>/dev/null | sed 's/^/ /' || echo " (Preview unavailable)" + jq '.mcpServers | to_entries[] | {name: .key, command: .value.command, args: .value.args, hasAuth: (.value.env.USABLE_API_TOKEN != null)}' .mcp.json 2>/dev/null | sed 's/^/ /' || echo " (Preview unavailable)" fi echo "" From 1b29e062d01ceef081008decbeb822316c21b381 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 12:58:07 +0100 Subject: [PATCH 33/41] feat(action): refresh MCP cache before validation to enhance reliability - Added a step to refresh the MCP cache prior to running validations, ensuring that the latest configurations are utilized. - Implemented a timeout for the cache refresh process to handle potential delays gracefully, improving overall robustness in the validation workflow. --- action.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/action.yml b/action.yml index d213a83..d4a8fbc 100644 --- a/action.yml +++ b/action.yml @@ -345,6 +345,13 @@ runs: forge config set --model "$MODEL" || echo "Warning: Could not set model config" fi + # Refresh MCP cache before validation + echo "๐Ÿ”„ Refreshing MCP cache..." + set +e + timeout 30 forge mcp cache refresh 2>&1 || echo "โš ๏ธ MCP cache refresh timed out or failed (continuing anyway)" + set -e + echo "" + # Run ForgeCode directly with MCP stdio transport echo "๐Ÿค– Running ForgeCode CLI with MCP stdio transport" echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" From e3205acb8b7d918ff2abee3d15b95199c77782ea Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 13:56:54 +0100 Subject: [PATCH 34/41] feat(scripts): enhance MCP connection testing with cache refresh and stdio transport - Updated the MCP connection testing script to include a cache refresh step before validation, improving reliability by ensuring the latest configurations are used. - Changed the transport method to stdio via npx for better subprocess control and clearer output during testing. - Enhanced output messages to reflect the new transport method and added a timeout for the cache refresh process, improving robustness in CI environments. --- scripts/test-mcp-connection.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index 9e0f960..1baa4da 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -14,7 +14,7 @@ echo "โœ… .mcp.json file found" echo "" echo "Configuration:" # Show config without exposing token -jq '.mcpServers | to_entries[] | {name: .key, command: .value.command, hasToken: (.value.env.USABLE_API_TOKEN != null)}' .mcp.json +jq '.mcpServers | to_entries[] | {name: .key, command: .value.command, args: .value.args, hasToken: (.value.env.USABLE_API_TOKEN != null)}' .mcp.json echo "" echo "Creating MCP test prompt..." @@ -81,9 +81,15 @@ echo "Checking if ForgeCode can see MCP servers in config..." forge mcp list 2>&1 || echo "โš ๏ธ forge mcp list command not available or failed" echo "" -echo "Note: Using HTTP transport to ${USABLE_URL}/api/mcp" -echo " This avoids stdio subprocess issues in CI environments" -echo " Authorization is passed via env.AUTHORIZATION in .mcp.json" +echo "๐Ÿ”„ Refreshing MCP cache..." +set +e +timeout 30 forge mcp cache refresh 2>&1 || echo "โš ๏ธ MCP cache refresh timed out or failed (continuing anyway)" +set -e + +echo "" +echo "Note: Using stdio transport via npx @usabledev/mcp-server" +echo " Direct subprocess communication for better control" +echo " Authentication passed via USABLE_API_TOKEN environment variable" echo "" echo "Running ForgeCode with MCP test prompt..." @@ -98,8 +104,8 @@ echo "Running ForgeCode with --verbose flag..." forge --verbose -p "$(cat /tmp/mcp-test-prompt.txt)" 2>&1 | tee /tmp/mcp-test-output.txt & FORGE_PID=$! -# Wait for ForgeCode to initialize and connect to HTTP MCP server -echo "Waiting for ForgeCode to connect to HTTP MCP server..." +# Wait for ForgeCode to initialize and connect to MCP server via stdio +echo "Waiting for ForgeCode to initialize MCP stdio connection..." sleep 5 # Check for any MCP-related patterns in the output From 33762046ef9b0185e46944eb9ba723269a0ab8ed Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 14:06:38 +0100 Subject: [PATCH 35/41] feat(action): improve MCP cache refresh handling for better reliability - Enhanced the MCP cache refresh logic to provide clearer output messages based on the success or failure of the refresh operation. - Implemented specific handling for timeout scenarios, ensuring that users are informed if the cache refresh times out or fails, which improves the robustness of the validation process. - Added a step to explicitly register the MCP server with ForgeCode during setup, enhancing the configuration process and ensuring better integration. --- action.yml | 11 ++++++++++- scripts/setup-mcp.sh | 11 ++++++++++- scripts/test-mcp-connection.sh | 11 ++++++++++- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/action.yml b/action.yml index d4a8fbc..b2512a9 100644 --- a/action.yml +++ b/action.yml @@ -348,7 +348,16 @@ runs: # Refresh MCP cache before validation echo "๐Ÿ”„ Refreshing MCP cache..." set +e - timeout 30 forge mcp cache refresh 2>&1 || echo "โš ๏ธ MCP cache refresh timed out or failed (continuing anyway)" + if timeout 30 forge mcp cache refresh 2>&1; then + echo "โœ… MCP cache refreshed successfully" + else + EXIT_CODE=$? + if [ $EXIT_CODE -eq 124 ]; then + echo "โš ๏ธ MCP cache refresh timed out after 30 seconds (continuing anyway)" + else + echo "โš ๏ธ MCP cache refresh failed (this is normal on first run - continuing anyway)" + fi + fi set -e echo "" diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index ce5f588..9ed618a 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -45,7 +45,6 @@ echo " Base URL: ${USABLE_URL}" # Verify the configuration if [ -f ".mcp.json" ]; then echo " Configuration file created successfully" - echo " MCP server 'usable-local' will be available to ForgeCode" # Show config (without exposing token) if command -v jq &> /dev/null; then @@ -57,6 +56,16 @@ if [ -f ".mcp.json" ]; then echo "" echo " Full .mcp.json content (with token masked):" cat .mcp.json | sed "s/$USABLE_API_TOKEN/***MASKED***/g" | sed 's/^/ /' || echo " (Could not read file)" + + # Register the MCP server explicitly with ForgeCode + echo "" + echo " Registering MCP server with ForgeCode..." + SERVER_CONFIG=$(jq -c '.mcpServers["usable-local"]' .mcp.json) + if forge mcp add-json usable-local "$SERVER_CONFIG" --scope local 2>&1; then + echo " โœ… MCP server 'usable-local' registered successfully" + else + echo " โš ๏ธ MCP server registration returned an error (it may already be registered from .mcp.json)" + fi else echo "::error::Failed to create .mcp.json" exit 1 diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index 1baa4da..36270a2 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -83,7 +83,16 @@ forge mcp list 2>&1 || echo "โš ๏ธ forge mcp list command not available or fai echo "" echo "๐Ÿ”„ Refreshing MCP cache..." set +e -timeout 30 forge mcp cache refresh 2>&1 || echo "โš ๏ธ MCP cache refresh timed out or failed (continuing anyway)" +if timeout 30 forge mcp cache refresh 2>&1; then + echo "โœ… MCP cache refreshed successfully" +else + EXIT_CODE=$? + if [ $EXIT_CODE -eq 124 ]; then + echo "โš ๏ธ MCP cache refresh timed out after 30 seconds (continuing anyway)" + else + echo "โš ๏ธ MCP cache refresh failed (this is normal on first run - continuing anyway)" + fi +fi set -e echo "" From f7ebfa3ff990b88e0d07f4e7a37fd25c9cf6e786 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 14:35:55 +0100 Subject: [PATCH 36/41] feat(action, scripts): enhance MCP server registration and cleanup process - Improved the MCP server registration logic by removing any existing server instance before adding a new one, preventing conflicts. - Updated the cleanup process to include the removal of MCP server registration, ensuring sensitive data is handled securely. - Enhanced output messages for better clarity during the registration and cleanup phases, improving user experience and debugging capabilities. --- action.yml | 7 ++-- scripts/setup-mcp.sh | 60 ++++++++++++---------------------- scripts/test-mcp-connection.sh | 18 +++++----- 3 files changed, 35 insertions(+), 50 deletions(-) diff --git a/action.yml b/action.yml index b2512a9..cd18f21 100644 --- a/action.yml +++ b/action.yml @@ -498,8 +498,11 @@ runs: # Remove temporary files containing sensitive data rm -f /tmp/forge-config.yaml rm -f /tmp/validation-*.md - rm -f .mcp.json # Remove MCP config with authentication token - echo "โœ… Temporary files cleaned up" + + # Remove MCP server registration (contains authentication token) + forge mcp remove usable-local 2>/dev/null || true + + echo "โœ… Temporary files and MCP registration cleaned up" - name: Fail on Critical Violations if: inputs.fail-on-critical == 'true' && steps.validate.outputs.validation_passed == 'false' diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index 9ed618a..2f7d8c9 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -15,59 +15,39 @@ USABLE_URL="${USABLE_URL:-https://usable.dev}" echo "Usable URL: $USABLE_URL" -# Write .mcp.json file with stdio transport using npx -# ForgeCode loads MCP servers from .mcp.json in the current directory +# Register MCP server with ForgeCode using stdio transport via npx echo "" -echo "Writing .mcp.json configuration..." +echo "Registering MCP server with ForgeCode..." -# Create the .mcp.json file with stdio transport using npx -# This uses the Usable MCP server package via npx for better subprocess control -cat > .mcp.json < /dev/null; then - echo "" - echo " Configuration preview:" - jq '.mcpServers | to_entries[] | {name: .key, command: .value.command, args: .value.args, hasAuth: (.value.env.USABLE_API_TOKEN != null)}' .mcp.json 2>/dev/null | sed 's/^/ /' || echo " (Preview unavailable)" - fi - - echo "" - echo " Full .mcp.json content (with token masked):" - cat .mcp.json | sed "s/$USABLE_API_TOKEN/***MASKED***/g" | sed 's/^/ /' || echo " (Could not read file)" - - # Register the MCP server explicitly with ForgeCode +# Remove existing server if it exists (to avoid conflicts) +forge mcp remove usable-local 2>/dev/null || true + +# Register the server +if forge mcp add-json usable-local "$SERVER_CONFIG" --scope local 2>&1; then + echo "โœ… MCP server 'usable-local' registered successfully" echo "" - echo " Registering MCP server with ForgeCode..." - SERVER_CONFIG=$(jq -c '.mcpServers["usable-local"]' .mcp.json) - if forge mcp add-json usable-local "$SERVER_CONFIG" --scope local 2>&1; then - echo " โœ… MCP server 'usable-local' registered successfully" - else - echo " โš ๏ธ MCP server registration returned an error (it may already be registered from .mcp.json)" - fi + echo "Verifying registration..." + forge mcp list 2>&1 | grep -i "usable-local" || echo "โš ๏ธ Server registered but not shown in list" else - echo "::error::Failed to create .mcp.json" + echo "::error::Failed to register MCP server with ForgeCode" exit 1 fi diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index 36270a2..865f0f4 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -3,18 +3,21 @@ set -euo pipefail echo "::group::Testing MCP Connection" -# Check if .mcp.json exists -if [ ! -f ".mcp.json" ]; then - echo "::error::.mcp.json file not found" - echo "MCP server configuration is missing" +# Check if MCP server is registered with ForgeCode +echo "Checking registered MCP servers..." +if ! forge mcp list 2>&1 | grep -q "usable-local"; then + echo "::error::MCP server 'usable-local' not found in ForgeCode configuration" + echo "Available servers:" + forge mcp list 2>&1 || echo " (none)" + echo "" + echo "Please run setup-mcp.sh first to register the MCP server" exit 1 fi -echo "โœ… .mcp.json file found" +echo "โœ… MCP server 'usable-local' is registered" echo "" echo "Configuration:" -# Show config without exposing token -jq '.mcpServers | to_entries[] | {name: .key, command: .value.command, args: .value.args, hasToken: (.value.env.USABLE_API_TOKEN != null)}' .mcp.json +forge mcp get usable-local 2>&1 || echo " (Unable to retrieve configuration)" echo "" echo "Creating MCP test prompt..." @@ -103,7 +106,6 @@ echo " Authentication passed via USABLE_API_TOKEN environment variable" echo "" echo "Running ForgeCode with MCP test prompt..." echo "Current working directory: $(pwd)" -echo ".mcp.json exists in PWD: $([ -f .mcp.json ] && echo 'YES' || echo 'NO')" echo "" echo "Capturing both stdout and stderr to see MCP initialization..." From ffbd1f4a823709fbbe5b5a38e6f20f6ecd7d29e3 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 14:52:30 +0100 Subject: [PATCH 37/41] feat(action, scripts): improve MCP cache refresh output handling - Enhanced the MCP cache refresh logic in both action and testing scripts to provide clearer output messages based on the success or failure of the refresh operation. - Implemented specific handling for timeout scenarios and cases where the cache directory does not exist, improving user awareness during the process. - Streamlined error reporting to include the first error message encountered during cache refresh failures, aiding in troubleshooting and debugging. --- action.yml | 19 +++++++++++-------- scripts/test-mcp-connection.sh | 19 +++++++++++-------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/action.yml b/action.yml index cd18f21..e40839e 100644 --- a/action.yml +++ b/action.yml @@ -348,17 +348,20 @@ runs: # Refresh MCP cache before validation echo "๐Ÿ”„ Refreshing MCP cache..." set +e - if timeout 30 forge mcp cache refresh 2>&1; then + CACHE_OUTPUT=$(timeout 30 forge mcp cache refresh 2>&1) + CACHE_EXIT=$? + set -e + + if [ $CACHE_EXIT -eq 0 ]; then echo "โœ… MCP cache refreshed successfully" + elif [ $CACHE_EXIT -eq 124 ]; then + echo "โš ๏ธ MCP cache refresh timed out after 30 seconds (continuing anyway)" + elif echo "$CACHE_OUTPUT" | grep -q "No such file or directory"; then + echo "โš ๏ธ No cache directory exists yet (first run) - skipping cache refresh" else - EXIT_CODE=$? - if [ $EXIT_CODE -eq 124 ]; then - echo "โš ๏ธ MCP cache refresh timed out after 30 seconds (continuing anyway)" - else - echo "โš ๏ธ MCP cache refresh failed (this is normal on first run - continuing anyway)" - fi + echo "โš ๏ธ MCP cache refresh failed - continuing anyway" + echo " Error: $(echo "$CACHE_OUTPUT" | grep -i "error" | head -1)" fi - set -e echo "" # Run ForgeCode directly with MCP stdio transport diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index 865f0f4..330bdf6 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -86,17 +86,20 @@ forge mcp list 2>&1 || echo "โš ๏ธ forge mcp list command not available or fai echo "" echo "๐Ÿ”„ Refreshing MCP cache..." set +e -if timeout 30 forge mcp cache refresh 2>&1; then +CACHE_OUTPUT=$(timeout 30 forge mcp cache refresh 2>&1) +CACHE_EXIT=$? +set -e + +if [ $CACHE_EXIT -eq 0 ]; then echo "โœ… MCP cache refreshed successfully" +elif [ $CACHE_EXIT -eq 124 ]; then + echo "โš ๏ธ MCP cache refresh timed out after 30 seconds (continuing anyway)" +elif echo "$CACHE_OUTPUT" | grep -q "No such file or directory"; then + echo "โš ๏ธ No cache directory exists yet (first run) - skipping cache refresh" else - EXIT_CODE=$? - if [ $EXIT_CODE -eq 124 ]; then - echo "โš ๏ธ MCP cache refresh timed out after 30 seconds (continuing anyway)" - else - echo "โš ๏ธ MCP cache refresh failed (this is normal on first run - continuing anyway)" - fi + echo "โš ๏ธ MCP cache refresh failed - continuing anyway" + echo " Error: $(echo "$CACHE_OUTPUT" | grep -i "error" | head -1)" fi -set -e echo "" echo "Note: Using stdio transport via npx @usabledev/mcp-server" From 4e4b8dd9c36a3b5566e93ee13b4196050fd44389 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 15:15:23 +0100 Subject: [PATCH 38/41] fix(scripts): update MCP server registration naming for consistency - Changed the MCP server registration name from 'usable-local' to 'usable' to align with naming conventions. - Updated related output messages to reflect the new server name, ensuring clarity during the registration process. - This change helps maintain consistency in server naming across the setup script. --- scripts/setup-mcp.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index 2f7d8c9..a0e48a4 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -38,14 +38,14 @@ echo " Base URL: ${USABLE_URL}" echo "" # Remove existing server if it exists (to avoid conflicts) -forge mcp remove usable-local 2>/dev/null || true +forge mcp remove usable 2>/dev/null || true # Register the server -if forge mcp add-json usable-local "$SERVER_CONFIG" --scope local 2>&1; then - echo "โœ… MCP server 'usable-local' registered successfully" +if forge mcp add-json usable "$SERVER_CONFIG" --scope local 2>&1; then + echo "โœ… MCP server 'usable' registered successfully" echo "" echo "Verifying registration..." - forge mcp list 2>&1 | grep -i "usable-local" || echo "โš ๏ธ Server registered but not shown in list" + forge mcp list 2>&1 | grep -i "usable" || echo "โš ๏ธ Server registered but not shown in list" else echo "::error::Failed to register MCP server with ForgeCode" exit 1 From 65a047c6c1bcf5535ee93b6befb410acda821ad1 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 15:29:06 +0100 Subject: [PATCH 39/41] fix(action, scripts): update MCP server name for consistency - Changed the MCP server name from 'usable-local' to 'usable' in both action and testing scripts to ensure uniformity. - Updated output messages to reflect the new server name, enhancing clarity during the connection testing and cleanup processes. - This change aligns with previous updates to maintain consistent naming conventions across the codebase. --- action.yml | 2 +- scripts/test-mcp-connection.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/action.yml b/action.yml index e40839e..7919420 100644 --- a/action.yml +++ b/action.yml @@ -503,7 +503,7 @@ runs: rm -f /tmp/validation-*.md # Remove MCP server registration (contains authentication token) - forge mcp remove usable-local 2>/dev/null || true + forge mcp remove usable 2>/dev/null || true echo "โœ… Temporary files and MCP registration cleaned up" diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index 330bdf6..0858bdf 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -5,8 +5,8 @@ echo "::group::Testing MCP Connection" # Check if MCP server is registered with ForgeCode echo "Checking registered MCP servers..." -if ! forge mcp list 2>&1 | grep -q "usable-local"; then - echo "::error::MCP server 'usable-local' not found in ForgeCode configuration" +if ! forge mcp list 2>&1 | grep -q "usable"; then + echo "::error::MCP server 'usable' not found in ForgeCode configuration" echo "Available servers:" forge mcp list 2>&1 || echo " (none)" echo "" @@ -14,10 +14,10 @@ if ! forge mcp list 2>&1 | grep -q "usable-local"; then exit 1 fi -echo "โœ… MCP server 'usable-local' is registered" +echo "โœ… MCP server 'usable' is registered" echo "" echo "Configuration:" -forge mcp get usable-local 2>&1 || echo " (Unable to retrieve configuration)" +forge mcp get usable 2>&1 || echo " (Unable to retrieve configuration)" echo "" echo "Creating MCP test prompt..." From 804107641bd021a9389c9a06981af32c6155e6a0 Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 15:34:43 +0100 Subject: [PATCH 40/41] fix(scripts): add delay before starting ForgeCode for MCP server readiness - Introduced a 5-second wait before initiating ForgeCode to ensure that the MCP server registration is fully settled. - This change aims to improve the reliability of the connection testing process by allowing sufficient time for server initialization. --- scripts/test-mcp-connection.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/test-mcp-connection.sh b/scripts/test-mcp-connection.sh index 0858bdf..19da524 100755 --- a/scripts/test-mcp-connection.sh +++ b/scripts/test-mcp-connection.sh @@ -115,6 +115,8 @@ echo "Capturing both stdout and stderr to see MCP initialization..." # Run ForgeCode with the test prompt, capturing all output including stderr # Use --verbose flag for detailed output echo "Running ForgeCode with --verbose flag..." +echo "Waiting 5 seconds before starting ForgeCode to ensure MCP server registration is settled..." +sleep 5 forge --verbose -p "$(cat /tmp/mcp-test-prompt.txt)" 2>&1 | tee /tmp/mcp-test-output.txt & FORGE_PID=$! From c7d98f0007065bec0b473d4efbc42d9729a42aec Mon Sep 17 00:00:00 2001 From: flowcore-platform Date: Thu, 23 Oct 2025 15:43:40 +0100 Subject: [PATCH 41/41] feat(config): add MCP server configuration to forge.yaml and update setup script messages - Introduced MCP server configuration in forge.yaml, specifying command and environment variables for the Usable MCP server. - Updated setup-mcp.sh to clarify the MCP server configuration process and the use of environment variables, enhancing user awareness during setup. - Added a backup method for registering the MCP server with ForgeCode CLI, ensuring robustness in server initialization. --- forge.yaml | 10 +++++++++- scripts/setup-mcp.sh | 10 +++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/forge.yaml b/forge.yaml index e0ec477..cc03a6a 100644 --- a/forge.yaml +++ b/forge.yaml @@ -1,2 +1,10 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/antinomyhq/forge/refs/heads/main/forge.schema.json -{} +mcpServers: + usable: + command: npx + args: + - "@usabledev/mcp-server@latest" + - server + env: + USABLE_API_TOKEN: ${USABLE_API_TOKEN} + USABLE_BASE_URL: ${USABLE_URL} diff --git a/scripts/setup-mcp.sh b/scripts/setup-mcp.sh index a0e48a4..11d371c 100755 --- a/scripts/setup-mcp.sh +++ b/scripts/setup-mcp.sh @@ -14,10 +14,14 @@ fi USABLE_URL="${USABLE_URL:-https://usable.dev}" echo "Usable URL: $USABLE_URL" - -# Register MCP server with ForgeCode using stdio transport via npx echo "" -echo "Registering MCP server with ForgeCode..." +echo "NOTE: MCP server is configured in forge.yaml" +echo " Environment variables USABLE_API_TOKEN and USABLE_URL will be used" +echo " ForgeCode will automatically load the MCP server from project config" +echo "" + +# Register MCP server with ForgeCode using stdio transport via npx (backup method) +echo "Registering MCP server with ForgeCode CLI as backup..." # Build the server configuration JSON SERVER_CONFIG=$(cat <