diff --git a/.env.example b/.env.example index a421f50..ae64fdc 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,7 @@ -OPENAI_API_KEY= # get from 1Password -GOOGLE_GENAI_API_KEY= #can be personal since google has a generous free tier> -ANTHROPIC_API_KEY= # get from Claude Console -SENTRY_DSN= # Currently not needed \ No newline at end of file +# API Keys for AI providers +OPENAI_API_KEY=sk-... +ANTHROPIC_API_KEY=sk-ant-... +GOOGLE_GENAI_API_KEY=... + +# Sentry Configuration (optional) +SENTRY_DSN= diff --git a/.gitignore b/.gitignore index 6e1c9a5..26bab56 100644 --- a/.gitignore +++ b/.gitignore @@ -55,4 +55,8 @@ Thumbs.db coverage/ .nyc_output/ test-results/ -*.log \ No newline at end of file +*.log + +# Test execution directory +runs/ +.secrets diff --git a/CLAUDE.md b/CLAUDE.md index 5614ae7..7eebb2c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,575 +10,603 @@ This repository contains a comprehensive testing framework for Sentry's AI SDK i 2. **Comprehensive coverage** - Test all popular AI SDKs that Sentry supports 3. **Language parity** - Identical test behavior across JavaScript and Python 4. **Clear error messages** - When tests fail, show exactly what's wrong -5. **Formal specification** - Language-agnostic JSON fixtures define expected behavior +5. **Template-based test generation** - Nunjucks templates generate runnable test files for each framework ## Architecture Overview +This project uses a **template-based test generation approach**. Test definitions (TypeScript) combined with framework templates (Nunjucks) generate runnable test files. A span collector HTTP server captures Sentry data for validation. + ### Project Structure ``` -ai-sdks-test/ -├── sdks/ -│ ├── js/ # JavaScript SDK implementations -│ │ ├── _test-utils/ # JS test utilities (CRITICAL: Keep in sync with py/) -│ │ │ ├── test-runner.cjs # Orchestrates test execution -│ │ │ ├── fixture-loader.cjs # Loads JSON fixtures with overrides -│ │ │ ├── validator.cjs # Validates captured spans against fixtures -│ │ │ └── mock-transport.cjs # Captures Sentry data in-memory -│ │ ├── vercel/ # Each SDK has its own directory -│ │ │ ├── setup.js # SDK-specific setup -│ │ │ ├── config.json # SDK configuration (framework type, overrides) -│ │ │ └── cases/ # Test cases (1-simple.js, etc.) -│ │ ├── openai/ -│ │ │ ├── setup.js -│ │ │ ├── config.json -│ │ │ └── cases/ -│ │ └── anthropic/ -│ │ ├── setup.js -│ │ ├── config.json -│ │ └── cases/ -│ └── py/ # Python SDK implementations -│ ├── _test-utils/ # Python test utilities (CRITICAL: Keep in sync with js/) -│ │ ├── test_runner.py # Orchestrates test execution -│ │ ├── fixture_loader.py # Loads JSON fixtures with overrides -│ │ ├── validator.py # Validates captured spans against fixtures -│ │ └── mock_transport.py # Captures Sentry data in-memory -│ ├── openai-agents/ -│ │ ├── setup.py # SDK-specific setup -│ │ ├── config.json # SDK configuration (framework type, overrides) -│ │ └── cases/ # Test cases (1-simple.py, etc.) -│ └── google-genai/ -│ ├── setup.py -│ ├── config.json -│ └── cases/ -├── shared/ -│ ├── specs/ # Test specifications and expectations -│ │ ├── sdk-config-schema.json # Schema for SDK config.json files -│ │ ├── 1-simple/ # Each spec in its own folder -│ │ │ ├── spec.md # Test specification document -│ │ │ ├── fixture-agentic.json # Expected spans for agentic frameworks -│ │ │ └── fixture-low-level.json # Expected spans for low-level frameworks -│ │ ├── 2-multi-step/ -│ │ │ └── spec.md # (fixture files not yet created) -│ │ └── ... (specs 3-8) # Additional specs (fixtures not yet created) -│ └── orchestration/ # Test runner (TypeScript) -│ ├── js-test-runner.cjs # Runner for JS tests -│ ├── python-test-runner.py # Runner for Python tests -│ ├── tsconfig.json # TypeScript configuration -│ ├── src/ # TypeScript source files (ES modules) -│ │ ├── cli.ts # Main CLI entry point -│ │ ├── runner.ts # Runs tests for both JS and Python -│ │ ├── discovery.ts # Discovers SDKs and test cases -│ │ ├── setup.ts # Setup utilities -│ │ ├── upgrade.ts # Upgrade utilities -│ │ ├── types.ts # Type definitions -│ │ └── reporters/ # Test result reporting -│ │ ├── console-printer.ts -│ │ ├── ctrf-generator.ts -│ │ └── html-generator.ts -│ ├── dist/ # Compiled JavaScript (ES modules) -│ │ └── ... -│ └── test-results/ # Generated test reports -│ ├── ctrf-report.json -│ └── test-report.html +testing-ai-sdk-integrations/ +├── src/ # TypeScript source code (ES modules) +│ ├── cli.ts # CLI entry point +│ ├── orchestrator.ts # Main test coordinator +│ ├── types.ts # Core type definitions +│ ├── validator.ts # Test validation logic +│ ├── setup.ts # Setup utilities +│ ├── concurrency.ts # Parallel execution support +│ ├── test-cases/ # Test definitions +│ │ ├── index.ts # Test registry +│ │ ├── checks.ts # Reusable check functions +│ │ ├── utils.ts # Test utilities (skip, assertions) +│ │ ├── llm/ # LLM test cases +│ │ │ ├── basic.ts # Basic single completion test +│ │ │ ├── multi-turn.ts # Multi-turn conversation test +│ │ │ ├── basic-error.ts # Error handling test +│ │ │ ├── vision.ts # Vision/image input test +│ │ │ └── long-input.ts # Long input trimming test +│ │ └── agents/ # Agent test cases +│ │ ├── basic.ts # Basic agent (no tools) +│ │ ├── tool-call.ts # Agent with tool calling +│ │ ├── tool-error.ts # Tool error handling +│ │ ├── vision.ts # Vision agent test +│ │ └── long-input.ts # Long input agent test +│ ├── runner/ # Test execution +│ │ ├── runner.ts # Main runner +│ │ ├── javascript-runner.ts # JS-specific execution +│ │ ├── python-runner.ts # Python-specific execution +│ │ ├── framework-config.ts # Framework configuration types +│ │ ├── framework-discovery.ts # Auto-discovers frameworks +│ │ ├── template-renderer.ts # Nunjucks template rendering +│ │ └── templates/ # Framework templates (see below) +│ ├── span-collector/ # HTTP server to capture Sentry data +│ │ ├── server.ts # Hono HTTP server +│ │ └── store.ts # In-memory span storage +│ └── reporters/ # Test output reporters +│ ├── ctrf-reporter.ts # CTRF JSON report generator +│ └── live-status.ts # Real-time test status display +├── dist/ # Compiled JavaScript output +├── runs/ # Generated test files per run +├── test-results/ # Generated reports +│ ├── ctrf-report-*.json +│ └── test-report-*.html +├── docs/ # Documentation +└── package.json ``` -## 📚 Documentation Navigation - -This is the main context file. For detailed guides, see: - -| Documentation | Purpose | Link | -| ------------------------------ | --------------------------------------------------------------------------- | ---------------------------------------------------------------- | -| **🔧 Adding SDKs** | Step-by-step guide for implementing new SDK tests with copy-paste templates | [sdks/README.md](sdks/README.md) | -| **📋 Test Specifications** | Fixture format, framework types, and spec system | [shared/specs/README.md](shared/specs/README.md) | -| **🧪 Test Utilities (JS)** | Mock transport, fixture validation, SDK helpers | [sdks/js/\_test-utils/README.md](sdks/js/_test-utils/README.md) | -| **🧪 Test Utilities (Python)** | Mock transport, fixture validation, SDK helpers | [sdks/py/\_test-utils/README.md](sdks/py/_test-utils/README.md) | -| **⚙️ CLI & Orchestration** | Running tests, test discovery, and debugging execution | [shared/orchestration/README.md](shared/orchestration/README.md) | -| **🐛 Troubleshooting** | Common pitfalls, error messages, and debugging tips | [docs/TROUBLESHOOTING.md](docs/TROUBLESHOOTING.md) | - -**Quick links:** - -- 🚀 Run all tests: `cd shared/orchestration && npm run cli run -- --all` -- 📝 List available SDKs: `npm run cli list` -- 🔍 Run specific SDK: `npm run cli run js/vercel` -- 💨 fail-fast: `npm run cli run -- --sdk js/openai --fail-fast` - -## Coding Standards & File Types - -### File Type Rules - -**JavaScript SDKs: Always use .js, NEVER .ts** - -- SDK implementations (`sdks/js/*/`) **must** use plain JavaScript files with `.js` extension -- Use CommonJS module system (`require()` and `module.exports`) -- **Reason:** Simplicity, compatibility, and to avoid TypeScript compilation complexity for SDK tests -- **Note:** The orchestrator uses TypeScript, but SDK implementations do not - -**Python SDKs: Always use .py** - -- SDK implementations (`sdks/py/*/`) use standard Python files with `.py` extension -- No type hints required (keep it simple) -- Use snake_case for all functions and variables (Python convention) - -### Module System Rules - -**Which module system to use where:** - -| Location | Module System | Syntax | File Extension | -| --------------------------------------- | -------------- | -------------------------------------------- | -------------- | -| SDK test files (`sdks/*/`) | **CommonJS** | `const X = require('...')`, `module.exports` | `.js` | -| Orchestration (`shared/orchestration/`) | **ES Modules** | `import X from '...'`, `export` | `.ts` | -| Test utilities (`sdks/js/_test-utils/`) | **CommonJS** | `const X = require('...')`, `module.exports` | `.cjs` | -| Python files | **Standard** | `import X`, `from X import Y` | `.py` | - -**Why these conventions?** +### Framework Templates Structure -- **CommonJS in SDKs:** Maximum compatibility and simplicity for contributors. No build step required, works directly with Node.js -- **TypeScript only in orchestration:** Type safety where complexity lives (test discovery, running, reporting). SDK tests are simple enough to not need TypeScript -- **Consistent patterns:** Makes copy-pasting templates easier and reduces cognitive load +Templates are organized by **category** (llm, agents), then **platform** (js, py), then **framework** name: -### File Naming Quick Reference - -| File Type | Pattern | Example | Location | -| ----------------- | ------------------------ | ------------------------ | ------------------------------ | -| Test spec | `{number}-{description}` | `1-simple` | `shared/specs/1-simple/` | -| JS test case | `{spec-id}.js` | `1-simple.js` | `sdks/js/vercel/cases/` | -| Python test case | `{spec-id}.py` | `1-simple.py` | `sdks/py/openai-agents/cases/` | -| JS SDK setup | `setup.js` | `setup.js` | `sdks/js/vercel/` | -| Python SDK setup | `setup.py` | `setup.py` | `sdks/py/openai-agents/` | -| Agentic fixture | `fixture-agentic.json` | `fixture-agentic.json` | `shared/specs/1-simple/` | -| Low-level fixture | `fixture-low-level.json` | `fixture-low-level.json` | `shared/specs/1-simple/` | - -### Import Paths & Module Resolution - -**JavaScript: Relative Paths** - -JavaScript test files use relative paths to import test utilities. **Count directory levels carefully:** - -```javascript -// From: sdks/js/vercel/cases/1-simple.js -// To: sdks/js/_test-utils/ - -const { runTestCase } = require("../../_test-utils/sdk-helpers.cjs"); -// ^^ -// 2 levels up: cases/ -> vercel/ -> js/_test-utils/ ``` - -**Path counting formula:** - -1. Start at your test file location -2. Count `../` for each directory level up -3. Then add the path to the target - -**Common paths from SDK files:** - -| From | To | Path | -| ------------------------------- | -------------------------- | ------------------------------------------ | -| `sdks/js/{sdk}/cases/{test}.js` | `sdks/js/_test-utils/` | `../../_test-utils/test-runner.cjs` | -| `sdks/js/{sdk}/setup.js` | `sdks/js/_test-utils/` | `../_test-utils/mock-transport.cjs` | -| `sdks/py/{sdk}/cases/{test}.py` | (uses sys.path, see below) | N/A - import directly after sys.path setup | - -**Python: sys.path Manipulation** - -Python SDKs must manually add the test utilities to `sys.path` because Python doesn't have a project-wide module resolution like Node.js. - -**Every Python SDK's `setup.py` MUST include this code:** - -```python -import sys -from pathlib import Path - -# Add test utils to path -test_utils_path = Path(__file__).parent.parent / "_test-utils" -sys.path.insert(0, str(test_utils_path)) - -# Now you can import directly -from test_runner import run_test_case -from mock_transport import create_mock_transport, get_mock_transport, clear_mock_transport +src/runner/templates/ +├── base.js.njk # Base JavaScript template +├── base.py.njk # Base Python template +├── llm/ # Low-level LLM frameworks +│ ├── js/ +│ │ ├── anthropic/ # config.json + template.njk +│ │ ├── google-genai/ +│ │ ├── langchain/ +│ │ └── openai/ +│ └── py/ +│ ├── anthropic/ +│ ├── langchain/ +│ ├── litellm/ +│ └── openai/ +└── agents/ # Agentic frameworks + ├── js/ + │ ├── langgraph/ + │ ├── mastra/ + │ └── vercel/ + └── py/ + ├── google-genai/ + ├── langgraph/ + ├── openai-agents/ + └── pydantic-ai/ ``` -**Why this is needed:** - -- Python's import system doesn't traverse up directories by default -- This adds the test utilities to the beginning of the module search path -- Must be done in `setup.py` before any test imports -- Test case files will inherit this path setup - -**When adding a new SDK:** - -- Copy the sys.path block from an existing Python SDK's `setup.py` -- Path is always `Path(__file__).parent.parent / "_test-utils"` (2 levels up) -- Test by running `python -c "from fixture_loader import load_fixture"` in your SDK directory - -## 🚨 CRITICAL: JavaScript/Python Parity Rule - -**The files in `sdks/js/_test-utils/` and `sdks/py/_test-utils/` MUST be kept perfectly synchronized.** - -### Why This Matters - -- Same fixtures (JSON) used by both languages -- Same validation logic = consistent behavior -- Same error messages = easier debugging -- Changes to one MUST be mirrored in the other - -### When You Change test-utils - -**ALWAYS update both JS and Python versions together:** - -1. **If you modify `js/_test-utils/validator.cjs`:** - - - Update `py/_test-utils/validator.py` with equivalent logic - - **Update `validator.test.cjs` and `validator.test.py` with test cases for the new feature** - - Run both test files: `node sdks/js/_test-utils/validator.test.cjs && python3 sdks/py/_test-utils/validator.test.py` - - Run same fixture through both validators - - Confirm identical error output - -2. **If you modify `js/_test-utils/test-runner.cjs`:** +## Quick Start - - Update `py/_test-utils/test_runner.py` with equivalent logic - - Test both implementations - - Verify error messages match - -3. **If you add a new helper function:** - - Implement in both languages - - Keep function signatures equivalent - - Document any language-specific differences - -### Files That Must Stay in Sync - -| JavaScript | Python | Purpose | -| ---------------------------------------- | --------------------------------------- | ---------------------------------------- | -| `sdks/js/_test-utils/test-runner.cjs` | `sdks/py/_test-utils/test_runner.py` | Orchestrates test execution | -| `sdks/js/_test-utils/fixture-loader.cjs` | `sdks/py/_test-utils/fixture_loader.py` | Loads JSON fixtures with overrides | -| `sdks/js/_test-utils/validator.cjs` | `sdks/py/_test-utils/validator.py` | Validates captured data against fixtures | -| `sdks/js/_test-utils/validator.test.cjs` | `sdks/py/_test-utils/validator.test.py` | Tests for validator logic | -| `sdks/js/_test-utils/mock-transport.cjs` | `sdks/py/_test-utils/mock_transport.py` | Captures Sentry events in-memory | - -### Test Parity Checklist - -When adding or modifying test-utils, verify: - -- [ ] Same function exists in both JS and Python -- [ ] Same parameters (accounting for language differences: camelCase vs snake_case) -- [ ] Same error messages (word-for-word when possible) -- [ ] Same return values/behavior -- [ ] Both implementations tested and working -- [ ] **Validator tests updated in both languages (validator.test.cjs and validator.test.py)** -- [ ] **Both test files pass: `node validator.test.cjs && python3 validator.test.py`** -- [ ] Same validation logic produces identical results -- [ ] Test with same fixture through both validators to confirm identical output - -### Current Parity Status - -| Component | JavaScript | Python | Status | Notes | -| ----------------- | -------------------- | ------------------- | --------- | ----------------------------------------- | -| Test Runner | `test-runner.cjs` | `test_runner.py` | ✅ Synced | Both orchestrate tests correctly | -| Mock Transport | `mock-transport.cjs` | `mock_transport.py` | ✅ Synced | Both capture envelopes correctly | -| Fixture Loader | `fixture-loader.cjs` | `fixture_loader.py` | ✅ Synced | Both support config overrides | -| Fixture Validator | `validator.cjs` | `validator.py` | ✅ Synced | Schema validation, pattern ops, wildcards | -| Validator Tests | `validator.test.cjs` | `validator.test.py` | ✅ Synced | Both test schema validation | - -## Test Scenarios - -### Current Test Cases +```bash +# Install dependencies +npm install -Test cases are identified by spec ID (e.g., "1-simple", "2-multi-step"). Each has: +# Build TypeScript +npm run build -- **JSON fixture(s)** in `shared/specs/{spec-id}/` defining expectations -- **JS implementation(s)** in `sdks/js/*/cases/` -- **Python implementation(s)** in `sdks/py/*/cases/` +# List all discovered frameworks +npm run test list -**Implemented:** +# Run all tests +npm run test run -- **1-simple**: Basic Completion - Single prompt with system message -- **2-multi-step**: Multi-step conversation - Two API calls with conversation history +# Run tests for a specific framework +npm run test -- --framework openai -**Planned:** +# Run tests for a specific platform +npm run test -- --platform py -- **3-agent-success**: Agentic workflow - success path -- **4-simple-with-error**: Basic completion with application error -- **5-streaming**: Basic streaming -- **6-streaming-with-error**: Streaming with application error -- **7-agent-llm-error**: Agentic workflow - error during LLM call -- **8-agent-tool-error**: Agentic workflow - error during tool execution +# Run with verbose output +npm run test -- --framework openai --verbose -### Sentry Features to Verify +# Run only streaming tests +npm run test -- --streaming -Each test must verify that Sentry captures: +# Run only sync tests (Python) +npm run test -- --platform py --sync -1. **Performance tracing** - Spans and transactions with proper timing -2. **AI monitoring data** - Model name, token counts, prompts, completions -3. **Error tracking** - Exceptions with context and stack traces (for error tests) +# Run tests in parallel +npm run test -- -j=4 -### Framework Types & Fixture Variants +# Setup only (generate test files without running) +npm run test setup -- --framework openai +``` -AI SDKs fall into two categories based on the span hierarchy they produce: +## CLI Reference -#### Agentic Frameworks +``` +Usage: + npm run test [command] [options] + +Commands: + run Run tests (default) + setup Setup environments and render templates (no test execution) + list List discovered frameworks + +Options: + --framework Filter by framework name + --test Filter by test name + --platform Filter by platform (js or py) + --sync Run only sync tests (default: both) + --async Run only async tests (default: both) + --streaming Run only streaming tests (default: both) + --blocking Run only blocking (non-streaming) tests (default: both) + --parallel, -j Run up to N tests in parallel (default: 1) + --verbose, -v Show detailed output (test execution logs, etc.) + --live-status Enable live status display (real-time tree view) + --open Open HTML report in browser after test run + --sentry-python Use local Sentry Python SDK (editable install) + --sentry-javascript Use local Sentry JavaScript SDK (link) + --help, -h Show this help message +``` -Frameworks that wrap LLM calls in agent abstraction spans: +## How Tests Work -- **Vercel AI SDK** (`js/vercel`) - Produces `gen_ai.invoke_agent` parent spans -- **OpenAI Agents SDK** (`py/openai-agents`) - Produces agent workflow spans +1. **Discovery**: `framework-discovery.ts` scans `templates/` directory for `config.json` files +2. **Matrix Generation**: Creates test matrix (framework x test definition x execution modes) +3. **Template Rendering**: Uses Nunjucks to generate runnable test files from templates +4. **Execution**: Runs generated tests with Sentry DSN pointing to span collector +5. **Validation**: Runs check functions from `checks` array against captured spans +6. **Reporting**: Generates console output + CTRF JSON + HTML reports -**Span hierarchy example:** +### Test Flow ``` -gen_ai.invoke_agent (parent) - └─ gen_ai.chat or gen_ai.generate_text (child) +TestDefinition (TypeScript) + Framework Template (Nunjucks) + ↓ + Template Renderer generates test file + ↓ + Runner executes test file + ↓ + Sentry SDK sends spans to Span Collector + ↓ + Validator runs checks array on captured spans + ↓ + Reporter outputs results ``` -#### Low-Level Frameworks - -Frameworks that directly produce LLM call spans without agent wrappers: - -- **OpenAI SDK** (`js/openai`) - Direct `gen_ai.chat` spans only -- **Anthropic SDK** (`js/anthropic`) - Direct LLM call spans -- **Google GenAI SDK** (`py/google-genai`) - Direct LLM call spans +## Supported AI SDKs -**Span hierarchy example:** +### Currently Implemented -``` -gen_ai.chat (no parent) +| Platform | SDK | Category | Type | Streaming | Execution Modes | +| ---------- | --------------- | -------- | -------- | --------- | --------------- | +| JavaScript | `openai` | llm | llm-only | both | - | +| JavaScript | `anthropic` | llm | llm-only | both | - | +| JavaScript | `google-genai` | llm | llm-only | both | - | +| JavaScript | `langchain` | llm | llm-only | both | - | +| JavaScript | `vercel` | agents | agentic | - | - | +| JavaScript | `langgraph` | agents | agentic | - | - | +| JavaScript | `mastra` | agents | agentic | - | - | +| Python | `openai` | llm | llm-only | both | sync/async | +| Python | `anthropic` | llm | llm-only | both | sync/async | +| Python | `langchain` | llm | llm-only | both | sync/async | +| Python | `litellm` | llm | llm-only | both | sync/async | +| Python | `openai-agents` | agents | agentic | - | async | +| Python | `langgraph` | agents | agentic | - | sync/async | +| Python | `pydantic-ai` | agents | agentic | - | async | +| Python | `google-genai` | agents | agentic | - | sync/async | + +## Test Cases + +Test cases are TypeScript files in `src/test-cases/` that define: + +- **name**: Human-readable test name +- **description**: What the test validates +- **type**: `"llm"` or `"agent"` (determines which frameworks can run it) +- **inputs**: Test input data (model, messages) +- **checks**: Array of check functions that validate captured spans + +### LLM Test Cases + +| Test | Description | +| ---------------------- | ----------------------------------------- | +| `Basic LLM Test` | Single completion with system message | +| `Multi Turn LLM Test` | Multi-turn conversation (3 turns) | +| `Basic Error LLM Test` | Tests API error handling | +| `Vision LLM Test` | Image input processing | +| `Long Input LLM Test` | Message trimming for large inputs (>20KB) | + +### Agent Test Cases + +| Test | Description | +| ----------------------- | --------------------------------------- | +| `Basic Agent Test` | Agent without tools (simple completion) | +| `Tool Call Agent Test` | Agent with successful tool calling | +| `Tool Error Agent Test` | Agent with tool that raises exception | +| `Vision Agent Test` | Agent that processes images | +| `Long Input Agent Test` | Agent with large input trimming | + +### Test Definition Example + +Test definitions use an explicit `checks` array with reusable check functions: + +```typescript +// src/test-cases/llm/basic.ts +import { TestDefinition } from "../../types.js"; +import { + checkAISpanCount, + checkChatSpanAttributes, + checkValidTokenUsage, + checkInputMessagesSchema, +} from "../checks.js"; + +export const basicLLMTest: TestDefinition = { + name: "Basic LLM Test", + description: "Single completion call with system message", + type: "llm", + + inputs: [ + { + model: "gpt-4o-mini", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "What is the capital of France?" }, + ], + }, + ], + + checks: [ + checkAISpanCount(1), + checkChatSpanAttributes, + checkValidTokenUsage, + checkInputMessagesSchema, + ], +}; ``` -#### Using Fixture Variants and SDK Config +## Check Functions -Each test case folder contains multiple fixture files to handle both framework types: +Reusable check functions are defined in `src/test-cases/checks.ts`. Each check is an object with a `name` and `fn`: -- `fixture-agentic.json` - Expects agent parent spans + LLM child spans -- `fixture-low-level.json` - Expects only direct LLM call spans +```typescript +interface Check { + name: string; + fn: (spans: CapturedSpan[], config: FrameworkConfig, testDef: TestDefinition) => void; +} +``` -**Framework type is configured per SDK in `config.json`:** +### Available Checks + +| Check | Description | +| ----------------------------- | ------------------------------------------------------- | +| `checkAISpanCount(n)` | Factory: validate AI span count (exact or min/max) | +| `checkChatSpanAttributes` | Validates chat/completion spans (model, messages) | +| `checkAgentSpanAttributes` | Validates agent invocation spans | +| `checkToolSpanAttributes` | Validates tool execution spans | +| `checkValidTokenUsage` | Token counts exist and are valid | +| `checkInputTokensCached` | Cached tokens ≤ input tokens | +| `checkOutputTokensReasoning` | Reasoning tokens ≤ output tokens | +| `checkInputMessagesSchema` | Validates message schema follows Sentry conventions | +| `checkAgentHierarchy` | Agent span hierarchy and name propagation | +| `checkAvailableTools` | Validates gen_ai.request.available_tools | +| `checkResponseToolCalls([])` | Factory: validate tool calls in LLM response | +| `checkToolCalls([])` | Factory: validate tool execution spans | +| `checkMessageTrimming` | Messages are trimmed below 15KB | +| `checkTrimmingMetadata` | Original length metadata is present | +| `checkBinaryRedaction` | Binary content (images) is redacted | + +## Framework Configuration + +Each framework has a `config.json` file that defines its capabilities: ```json { - "sdk_name": "vercel", - "framework_type": "agentic", - "overrides": {} + "name": "openai", + "displayName": "OpenAI JavaScript SDK", + "type": "llm-only", + "platform": "js", + "streamingMode": "both", + "dependencies": [{ "package": "openai", "version": "framework" }], + "versions": ["4.96.0"], + "sentryVersions": ["latest"] } ``` -Test cases automatically use the framework type from their SDK's `config.json`: +### Configuration Fields + +| Field | Description | +| ---------------- | ---------------------------------------------------- | +| `name` | Framework identifier | +| `displayName` | Human-readable name | +| `type` | `"llm-only"` or `"agentic"` | +| `platform` | `"js"` or `"py"` | +| `streamingMode` | `"streaming"`, `"blocking"`, or `"both"` | +| `executionMode` | Python only: `"sync"`, `"async"`, or `"both"` | +| `dependencies` | NPM/pip packages to install | +| `versions` | Framework versions to test | +| `sentryVersions` | Sentry SDK versions to test against | +| `modelOverrides` | Override model names for request/response validation | +| `skip` | Tests or checks to skip for this framework | + +## Test Utilities + +Available in `src/test-cases/utils.ts`: + +| Function | Purpose | +| ---------------------- | -------------------------------------- | +| `skip(reason)` | Skip the current check with a reason | +| `skipIf(cond, reason)` | Conditionally skip a check | +| `extractGenAISpans()` | Filter spans for `gen_ai.*` operations | +| `findAgentSpans()` | Find `invoke_agent` spans | +| `findChatSpans()` | Find `chat`/`completion` spans | +| `findToolSpans()` | Find tool execution spans | +| `assertAttributes()` | Schema-based attribute validation | +| `printSpanSummary()` | Debug helper to print captured spans | + +### Attribute Schema + +The `assertAttributes` function supports: + +- `true`: Attribute must exist (any value) +- `false`: Attribute must NOT exist +- `"pattern*"`: Wildcard pattern matching +- `"exact"` / `123`: Exact value match + +```typescript +assertAttributes(spans, { + "gen_ai.operation.name": true, // Must exist + "gen_ai.request.model": "gpt-4", // Exact match + "gen_ai.response.model": "gpt-4*", // Pattern match + sensitive_field: false, // Must NOT exist +}); +``` -**JavaScript:** +## Adding a New Framework -```javascript -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); -const { Sentry } = require("../setup"); +### 1. Create Template Directory -async function testLogic(inputs) { - // Your test implementation -} +```bash +mkdir -p src/runner/templates/{llm|agents}/{js|py}/your-framework +``` + +### 2. Create `config.json` -// Framework type loaded from config.json automatically -module.exports = runTestCase("1-simple", testLogic, Sentry); +```json +{ + "name": "your-framework", + "displayName": "Your Framework SDK", + "type": "llm-only", + "platform": "js", + "streamingMode": "both", + "dependencies": [{ "package": "your-framework", "version": "framework" }], + "versions": ["1.0.0"], + "sentryVersions": ["latest"] +} ``` -**Python:** +### 3. Create `template.njk` -```python -from test_runner import run_test_case +Templates extend the base template and implement required blocks: -async def test_logic(inputs): - # Your test implementation - pass +```njk +{% extends "base.js.njk" %} -# Framework type loaded from config.json automatically -test_case = run_test_case("1-simple", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] -``` +{% block setup %} +let client; +{% endblock %} -**Important:** Each SDK's `config.json` defines its framework type. All test cases in that SDK use the same framework type. +{% block dynamic_imports %} + const SDK = (await import("your-framework")).default; + client = new SDK(); +{% endblock %} -#### SDK Framework Type Mapping +{% block test %} +{% for input in inputs %} + const response = await client.complete({ + model: "{{ input.model }}", + messages: {{ input.messages | dump }}, + }); + console.log("Response:", response.content); +{% endfor %} +{% endblock %} +``` -**When adding a new SDK, determine its framework type first, then use the same type across all test cases for that SDK.** +### 4. Build and Test -| SDK Path | Framework Type | Reason | -| ------------------ | -------------- | ------------------------------------------- | -| `js/vercel` | `agentic` | Produces `gen_ai.invoke_agent` parent spans | -| `js/openai` | `low-level` | Direct `gen_ai.chat` spans only | -| `js/anthropic` | `low-level` | Direct LLM call spans only | -| `py/openai-agents` | `agentic` | Produces agent workflow spans | -| `py/google-genai` | `low-level` | Direct LLM call spans only | +```bash +npm run build +npm run test -- --framework your-framework --verbose +``` -**How to determine framework type for a new SDK:** +## Adding a New Test Case -1. Run a simple test case with the SDK -2. Examine the captured spans -3. If you see agent/workflow wrapper spans → `agentic` -4. If you only see direct LLM call spans → `low-level` +### 1. Create Test File -## How Tests Work +```typescript +// src/test-cases/llm/your-test.ts +import { TestDefinition } from "../../types.js"; +import { checkAISpanCount, checkChatSpanAttributes } from "../checks.js"; -**Overview:** Tests run AI SDK code instrumented with Sentry, capture events in-memory, and validate against JSON fixtures. +export const yourTest: TestDefinition = { + name: "Your Test Name", + description: "What this test validates", + type: "llm", // or 'agent' -**Test flow:** + inputs: [ + { + model: "gpt-4o-mini", + messages: [{ role: "user", content: "Test prompt" }], + }, + ], -1. Load fixture defining expected behavior -2. Run AI SDK code within Sentry transaction -3. Mock transport captures spans/events -4. Validator compares captured data vs fixture expectations -5. Clear error messages show exactly what's missing + checks: [ + checkAISpanCount({ min: 1 }), + checkChatSpanAttributes, + ], +}; -**Key components:** +export default yourTest; +``` -- **Fixtures** (`shared/specs/*/fixture-*.json`) - Define expected spans and attributes -- **Mock transport** - Captures Sentry data in-memory instead of sending to server -- **Validator** - Compares actual vs expected, shows clear diffs +### 2. Register in Index -For details, see: +```typescript +// src/test-cases/index.ts +import { yourTest } from "./llm/your-test.js"; -- [shared/specs/README.md](shared/specs/README.md) - Fixture format -- [shared/test-utils/README.md](shared/test-utils/README.md) - Mock transport and validation +export const testCases = { + llm: { + // ... existing tests + yourTest: yourTest, + }, +}; +``` -## Supported AI SDKs +### 3. Build and Test -### Currently Implemented +```bash +npm run build +npm run test -- --test "Your Test Name" --verbose +``` -| Language | SDK | Framework Type | Status | Test Cases | -| ---------- | --------------- | -------------- | ---------- | ---------- | -| JavaScript | `vercel` | agentic | ✅ Working | 1-simple | -| JavaScript | `openai` | low-level | ✅ Working | 1-simple | -| JavaScript | `anthropic` | low-level | ✅ Working | 1-simple | -| JavaScript | `langchain` | low-level | ✅ Working | 1-simple | -| JavaScript | `langgraph` | agentic | ✅ Working | 1-simple | -| JavaScript | `google-genai` | low-level | ✅ Working | 1-simple | -| Python | `openai` | low-level | ✅ Working | 1-simple | -| Python | `openai-agents` | agentic | ✅ Working | 1-simple | -| Python | `anthropic` | low-level | ✅ Working | 1-simple | -| Python | `langchain` | low-level | ✅ Working | 1-simple | -| Python | `langgraph` | agentic | ✅ Working | 1-simple | -| Python | `google-genai` | low-level | ✅ Working | 1-simple | -| Python | `litellm` | low-level | ✅ Working | 1-simple | -| Python | `pydantic-ai` | agentic | ✅ Working | 1-simple | +## Core Types -**Note:** Not all SDKs support all features (streaming, function calling, etc.) +### TestDefinition -## Adding a New SDK +```typescript +interface TestDefinition { + name: string; + description: string; + type: "llm" | "agent"; + inputs: TestInput[]; + agent?: AgentDefinition; // For agent tests + causeAPIError?: boolean; // Trigger API errors + checks: Check[]; // Array of check functions +} +``` -For detailed step-by-step guides on implementing new SDK tests, see: +### FrameworkConfig + +```typescript +interface FrameworkConfig { + name: string; + platform: "js" | "py"; + type: "llm-only" | "agentic"; + version: string; + sentryVersion: string; + templatePath?: string; + executionMode?: "sync" | "async" | "both"; + streamingMode?: "streaming" | "blocking" | "both"; + modelOverrides?: { request?: string; response?: string }; + skip?: { tests?: string[]; checks?: { [testName: string]: string[] } }; +} +``` -- **[sdks/README.md](sdks/README.md)** - Complete templates and instructions for JavaScript and Python SDKs +### CapturedSpan + +```typescript +interface CapturedSpan { + span_id: string; + trace_id: string; + op: string; + description?: string; + start_timestamp: number; + timestamp: number; + data?: Record; + tags?: Record; +} +``` -**Quick start:** +## Environment Variables -1. Determine framework type (agentic vs low-level) -2. Copy template from sdks/README.md -3. Implement test cases -4. Run: `npm run cli run {language}/{your-sdk}` +All API keys should be in a root `.env` file (gitignored): -## Current Status +```bash +# .env +OPENAI_API_KEY=sk-... +ANTHROPIC_API_KEY=sk-ant-... +GOOGLE_API_KEY=... +``` -**Status:** Foundation complete, 14 SDKs implemented, 2 test specs complete +## Debugging -**What's Working:** +### View Captured Spans -- ✅ Test orchestration (CLI, discovery, runner) -- ✅ Mock transports (JS and Python) -- ✅ Refactored validators (modular, maintainable, 89% smaller main function) -- ✅ Shared span definitions (`$ref` syntax, 50% less duplication) -- ✅ Schema validation (`json_array`, `plain_string` types) -- ✅ Pattern-based op matching with exclusions -- ✅ Order-based span matching (no occurrence field needed) -- ✅ Clear error messages (fixture ID + actual op shown) -- ✅ SDK configuration with overrides (config.json) -- ✅ Centralized configuration (root .env, fixture inputs) -- ✅ Test reporting (console, CTRF JSON, HTML) -- ✅ Flexible CLI filtering (language, partial name, exact path) +Use `printSpanSummary()` in your check methods: -**What's Next:** +```typescript +import { printSpanSummary } from "../utils.js"; -- Implement test cases 3-8 (error handling, streaming, agentic workflows) +const debugCheck: Check = { + name: "debugCheck", + fn: (spans) => { + printSpanSummary(spans); + }, +}; +``` -## Implementation Guidelines +### Verbose Mode -### Centralized Configuration +```bash +npm run test -- --framework openai --verbose +``` -**Environment variables:** All API keys in root `.env` file (gitignored): +### Live Status ```bash -# .env (at repository root) -OPENAI_API_KEY=sk-... -ANTHROPIC_API_KEY=sk-ant-... -SENTRY_DSN=https://... +npm run test -- --framework openai --live-status ``` -**Test inputs:** Defined in fixture JSON files (`shared/specs/*/fixture-*.json`): +### Setup Only (Inspect Generated Files) -```json -{ - "spec_id": "1-simple", - "inputs": { - "model": "gpt-5-nano", - "system": "You are a helpful assistant.", - "prompt": "What is 2+2?" - } -} +```bash +npm run test setup -- --framework openai +# Check runs/ directory for generated test files ``` -This keeps tests language-agnostic - same fixtures work for JS and Python. +## Sentry Features to Verify -### Success Criteria +Each test validates that Sentry captures: -A test passes when: +1. **Performance tracing** - Spans with proper timing and hierarchy +2. **AI monitoring data** - Model name, token counts, operation names +3. **Error tracking** - Exceptions with context (for error tests) +4. **Message handling** - Proper schema, trimming, binary redaction -1. ✅ Test code runs without exceptions -2. ✅ Sentry captures all expected spans (minimum count met) -3. ✅ Required attributes present on each span -4. ✅ Span hierarchy correct (parent-child relationships) -5. ✅ Expected number of errors/events captured +## Success Criteria -## Debugging & Troubleshooting +A test passes when: -For common issues, error messages, and debugging tips, see: +1. Test code runs without exceptions +2. All check functions pass (or are skipped with reason) +3. Required spans are captured with correct attributes -- **[docs/TROUBLESHOOTING.md](docs/TROUBLESHOOTING.md)** - Complete troubleshooting guide with solutions to 8 common pitfalls +## Special Frameworks -**Quick diagnostic checklist:** +### Mastra -- Did you create a `.venv` and install requirements (Python)? -- Are relative import paths correct? (count `../` carefully) -- Is `sys.path.insert(0, ...)` present in Python setup.py? -- Are you using `.js` files (not `.ts`) for SDK tests? -- Is `FRAMEWORK_TYPE` set correctly for your SDK? +Mastra uses its own Sentry integration (`@mastra/sentry`) rather than `@sentry/node`. Key differences: +- Uses `SentryExporter` with Mastra's `Observability` system +- Attribute names follow newer OpenTelemetry conventions (`gen_ai.input.messages` instead of `gen_ai.request.messages`) +- Template is standalone (does not extend base.js.njk) ## References - **Sentry JavaScript SDK:** https://github.com/getsentry/sentry-javascript - **Sentry Python SDK:** https://github.com/getsentry/sentry-python - **Vercel AI SDK:** https://sdk.vercel.ai/docs -- **OpenAI Python Agents:** https://github.com/openai/swarm (inspiration) - -## Development Workflow Summary - -1. **Adding a feature to test-utils?** - - - Implement in JS - - Implement equivalent in Python - - Test both - - Verify error messages match - -2. **Adding a new test case?** - - - Create folder in `shared/specs/{number}-{description}/` - - Add `spec.md` (specification) - - Add `fixture-agentic.json` and `fixture-low-level.json` (inputs + expectations) - - Implement in at least one JS SDK - - Implement in at least one Python SDK - - Run: `npm run cli run -- --case {number}-{description}` - -3. **Adding a new SDK?** - - - Create directory structure (see sdks/README.md for templates) - - Create `config.json` with framework type and overrides - - Implement `setup.js` or `setup.py` with Sentry initialization - - Implement test cases in `cases/` directory (start with 1-simple) - - Run: `npm run cli run {js|py}/your-sdk` - -4. **Debugging test failures?** - - Look at "Span's actual attributes" in error message - - Compare with "Required attributes" - - Adjust fixture or fix SDK instrumentation +- **OpenAI Python SDK:** https://github.com/openai/openai-python +- **Mastra AI Framework:** https://mastra.ai/docs diff --git a/HUMANS.md b/HUMANS.md index 12adf49..7260671 100644 --- a/HUMANS.md +++ b/HUMANS.md @@ -1,17 +1,16 @@ This repo (hopefully) contains everything needed to test Sentry SDK AI integrations for Python and JavaScript. -Quick start and other goodies can be found in (./README.md). +Quick start and other goodies can be found in [README.md](./README.md). -The entire repo was made with Claude Code, and all of the major changes (like refactorings, adding SDKs, etc.) should be done by an agent. Most directories contain README.md files that the agent is instructed to read and update when needed. `.claude/settings.json` make sure it can't read this file or your `.env` +The entire repo was made with Claude Code, and all of the major changes (like refactorings, adding SDKs, etc.) should be done by an agent. Most directories contain README.md files that the agent is instructed to read and update when needed. `.claude/settings.json` makes sure it can't read this file or your `.env` ### The Gist -- There is a separate project directory for every integration, ensuring they are independent of each other. -- The setup file contains code that should be executed before the tests and, in most cases, contains only Sentry SDK initialization with the correct options and mock transport. -- Every test performs real LLM calls. -- After it is done, the mock transport is used to extract all of the envelopes the SDK would send. -- The validator then extracts relevant spans and checks against the fixture. -- The result is reported in CTRF as JSON, HTML, and printed in the console. +- Test definitions (TypeScript) + framework templates (Nunjucks) = generated test files +- A span collector HTTP server acts as a mock Sentry endpoint +- Tests make real LLM calls and the Sentry SDK captures spans +- The validator runs check functions against captured spans +- Results are reported as CTRF JSON, HTML, and printed to the console #### What this can do: @@ -19,7 +18,10 @@ Assert that AI integrations: - correctly initialize - capture all relevant spans in correct order/hierarchy -- correctly capture available attributes +- correctly capture available attributes (model, tokens, messages, tool calls) +- properly handle streaming vs blocking modes +- properly handle sync vs async execution (Python) +- trim long messages and redact binary content #### What this can't do: @@ -30,22 +32,27 @@ Assert that: ### JS vs. Py -Some parts of the test logic are implemented twice (once for JS and once for Python). They can never be exactly the same, but it is vital that they are as close to each other as possible in terms of overall functionality, file names, function names, variable names, etc. +The test cases are defined once in TypeScript and then rendered for each framework using Nunjucks templates. While the templates differ between JS and Python, they aim to produce equivalent behavior. Framework-specific quirks are handled in templates and the `skip` configuration. ### Adding another AI SDK integration -- Should be a matter of prompting an agent to do so. -- Make sure to repeat that it should be consistent with the other SDKs. -- Double-check if it wrote BS tests just to have them pass. -- Make sure it DID NOT change any fixtures or validators to make the tests pass. -- Make sure the package versions are pinned. +- Should be a matter of prompting an agent to do so +- Make sure to repeat that it should be consistent with the other SDKs +- Double-check if it wrote BS tests just to have them pass +- Make sure it DID NOT change any check functions or skip configurations to make the tests pass +- Make sure the package versions are pinned in `config.json` ### Adding more test cases -- The fixture should be written and double-checked by a human. -- The case should be implemented for all SDKs where it makes sense. -- If needed, it can be split into 2 flavors - "agentic" and "low-level." +- The check functions should be written and double-checked by a human +- The case should be implemented for all SDKs where it makes sense +- Test cases are split by type: `llm` (low-level SDKs) and `agent` (agentic frameworks) +- Use existing check functions from `checks.ts` when possible +- Add new checks to `checks.ts` if they're reusable across tests ### Versioning -- Every SDK has an independent Sentry SDK installation. This can be using the CLI but if you do so, make sure that the bersion is bumped everywhere. +- Every framework has an independent Sentry SDK version specified in `config.json` +- The `sentryVersions` array can include specific versions or `"latest"` +- Framework versions are specified in the `versions` array +- Dependencies use `"framework"` as version to inherit from the framework version diff --git a/README.md b/README.md index 30cea6e..94266ee 100644 --- a/README.md +++ b/README.md @@ -14,66 +14,69 @@ Sentry SDKs (JavaScript and Python) automatically instrument popular AI SDKs lik ## Project Structure ``` -ai-sdks-test/ -├── sdks/ -│ ├── js/ # JavaScript SDK implementations -│ │ ├── _test-utils/ # JS test utilities (mock transport, fixtures, validators) -│ │ ├── openai/ -│ │ │ ├── setup.js # Sentry initialization -│ │ │ ├── config.json # SDK configuration (framework type, overrides) -│ │ │ ├── package.json -│ │ │ └── cases/ # Test case implementations -│ │ │ └── 1-simple.js -│ │ ├── anthropic/ -│ │ ├── langchain/ -│ │ ├── langgraph/ -│ │ ├── vercel/ -│ │ └── google-genai/ -│ └── py/ # Python SDK implementations -│ ├── _test-utils/ # Python test utilities (mock transport, fixtures, validators) -│ ├── openai/ -│ │ ├── setup.py # Sentry initialization -│ │ ├── config.json # SDK configuration (framework type, overrides) -│ │ ├── requirements.txt -│ │ └── cases/ # Test case implementations -│ │ └── 1-simple.py -│ ├── openai-agents/ -│ ├── anthropic/ -│ ├── langchain/ -│ ├── langgraph/ -│ ├── google-genai/ -│ ├── litellm/ -│ └── pydantic-ai/ -├── shared/ -│ ├── specs/ # Test specifications (language-agnostic) -│ │ ├── 1-simple/ -│ │ │ ├── spec.md # Human-readable specification -│ │ │ ├── fixture-agentic.json # Expected spans for agentic frameworks -│ │ │ └── fixture-low-level.json # Expected spans for low-level frameworks -│ │ └── 2-multi-step/ -│ └── orchestration/ # Test runner and CLI -│ ├── src/ # TypeScript source -│ │ ├── cli.ts # CLI entry point -│ │ ├── runner.ts # Test execution -│ │ ├── discovery.ts # SDK/test discovery -│ │ ├── setup.ts # Dependency installation -│ │ └── reporters/ # Test reporting (console, CTRF, HTML) -│ ├── dist/ # Compiled JavaScript -│ └── test-results/ # Generated test reports -├── .env # Environment variables (gitignored) -├── .env.example # Template for API keys -├── action.yml # GitHub Action to run the tests for a specific language on CI -└── package.json # Root package.json for CLI alias +testing-ai-sdk-integrations/ +├── src/ # TypeScript source code (ES modules) +│ ├── cli.ts # CLI entry point +│ ├── orchestrator.ts # Main test coordinator +│ ├── types.ts # Core type definitions +│ ├── validator.ts # Test validation logic +│ ├── setup.ts # Setup utilities +│ ├── concurrency.ts # Parallel execution support +│ ├── test-cases/ # Test definitions +│ │ ├── index.ts # Test registry +│ │ ├── checks.ts # Reusable check functions +│ │ ├── utils.ts # Test utilities (skip, assertions) +│ │ ├── llm/ # LLM test cases +│ │ │ ├── basic.ts # Basic single completion test +│ │ │ ├── multi-turn.ts # Multi-turn conversation test +│ │ │ ├── basic-error.ts # Error handling test +│ │ │ ├── vision.ts # Vision/image input test +│ │ │ └── long-input.ts # Long input trimming test +│ │ └── agents/ # Agent test cases +│ │ ├── basic.ts # Basic agent (no tools) +│ │ ├── tool-call.ts # Agent with tool calling +│ │ ├── tool-error.ts # Tool error handling +│ │ ├── vision.ts # Vision agent test +│ │ └── long-input.ts # Long input agent test +│ ├── runner/ # Test execution +│ │ ├── runner.ts # Main runner +│ │ ├── javascript-runner.ts # JS-specific execution +│ │ ├── python-runner.ts # Python-specific execution +│ │ ├── framework-config.ts # Framework configuration types +│ │ ├── framework-discovery.ts # Auto-discovers frameworks +│ │ ├── template-renderer.ts # Nunjucks template rendering +│ │ └── templates/ # Framework templates +│ │ ├── base.js.njk # Base JavaScript template +│ │ ├── base.py.njk # Base Python template +│ │ ├── llm/ # LLM framework templates +│ │ │ ├── js/{openai,anthropic,google-genai,langchain}/ +│ │ │ └── py/{openai,anthropic,langchain,litellm}/ +│ │ └── agents/ # Agent framework templates +│ │ ├── js/{langgraph,mastra,vercel}/ +│ │ └── py/{langgraph,openai-agents,pydantic-ai,google-genai}/ +│ ├── span-collector/ # HTTP server to capture Sentry data +│ │ ├── server.ts # Hono HTTP server +│ │ └── store.ts # In-memory span storage +│ └── reporters/ # Test output reporters +│ ├── ctrf-reporter.ts # CTRF JSON report generator +│ └── live-status.ts # Real-time test status display +├── dist/ # Compiled JavaScript output +├── runs/ # Generated test files per run +├── test-results/ # Generated reports +│ └── ctrf-report.json +├── .env # Environment variables (gitignored) +├── .env.example # Template for API keys +└── package.json ``` ## Quick Start ### Prerequisites -- Node.js 18+ (for JavaScript tests) +- Node.js 18+ (for JavaScript tests and orchestration) - Python 3.9+ (for Python tests) -- API keys for AI services (OpenAI, Anthropic, etc.) -- Sentry project DSN (for E2E tests) +- uv (Python package manager, recommended) +- API keys for AI services (OpenAI, Anthropic, Google) ### Setup @@ -88,108 +91,639 @@ cd testing-ai-sdk-integrations ```bash cp .env.example .env -# Edit .env with your API keys and Sentry DSN +# Edit .env with your API keys ``` -3. Install orchestration dependencies: +3. Install dependencies and build: ```bash -cd shared/orchestration npm install -cd ../.. +npm run build ``` -4. Set up all SDK dependencies: +4. List available frameworks: ```bash -npm run cli setup +npm run test list ``` 5. Run all tests: ```bash -npm run cli run -- --all +npm run test run ``` -6. Run tests with filters: +### CLI Usage ```bash -# All JavaScript SDKs -npm run cli run js +# Run all tests +npm run test run + +# Run tests for a specific framework +npm run test -- --framework openai + +# Run tests for a specific platform +npm run test -- --platform py + +# Run a specific test +npm run test -- --test "Basic LLM Test" + +# Run with verbose output +npm run test -- --framework openai --verbose + +# Run only streaming tests +npm run test -- --streaming + +# Run only blocking (non-streaming) tests +npm run test -- --blocking + +# Run only sync tests (Python) +npm run test -- --platform py --sync + +# Run only async tests (Python) +npm run test -- --platform py --async + +# Run tests in parallel +npm run test -- -j=4 + +# Run tests and open report in browser +npm run test -- --framework openai --open + +# Setup only (generate test files without running) +npm run test setup -- --framework openai + +# Use local Sentry SDK +npm run test -- --sentry-python /path/to/sentry-python +npm run test -- --sentry-javascript /path/to/sentry-javascript +``` + +### CLI Options + +| Option | Description | +| ---------------------------- | -------------------------------------------- | +| `--framework ` | Filter by framework name | +| `--test ` | Filter by test name | +| `--platform ` | Filter by platform | +| `--sync` | Run only sync tests (Python, default: both) | +| `--async` | Run only async tests (Python, default: both) | +| `--streaming` | Run only streaming tests (default: both) | +| `--blocking` | Run only blocking tests (default: both) | +| `-j, --parallel ` | Run up to N tests in parallel | +| `-v, --verbose` | Show detailed output | +| `--live-status` | Enable real-time status display | +| `--open` | Open HTML report in browser after test run | +| `--sentry-python ` | Use local Sentry Python SDK | +| `--sentry-javascript ` | Use local Sentry JavaScript SDK | + +## Test Matrix Structure + +Tests are organized in a hierarchical structure: + +``` +Type / Platform / Framework / Test Case +``` + +| Dimension | Description | Examples | +| ------------- | -------------------------- | --------------------------------------------------------- | +| **Type** | Category of AI integration | `llm` (low-level LLM SDKs), `agents` (agentic frameworks) | +| **Platform** | Programming language | `js` (JavaScript/Node.js), `py` (Python) | +| **Framework** | AI SDK being tested | `openai`, `anthropic`, `langchain`, `langgraph`, etc. | +| **Test Case** | Specific test scenario | `Basic LLM Test`, `Tool Call Agent Test`, etc. | + +This structure is reflected in the templates directory: + +``` +src/runner/templates/ +├── llm/ # Type: LLM +│ ├── js/ # Platform: JavaScript +│ │ ├── openai/ # Framework +│ │ ├── anthropic/ +│ │ ├── google-genai/ +│ │ └── langchain/ +│ └── py/ # Platform: Python +│ ├── openai/ +│ ├── anthropic/ +│ ├── langchain/ +│ └── litellm/ +└── agents/ # Type: Agents + ├── js/ + │ ├── langgraph/ + │ ├── mastra/ + │ └── vercel/ + └── py/ + ├── langgraph/ + ├── openai-agents/ + ├── pydantic-ai/ + └── google-genai/ +``` + +When tests run, each **Test Case** is rendered using the framework's template and executed. For example: + +- `llm / py / openai / Basic LLM Test` → Tests OpenAI Python SDK with a simple completion +- `agents / js / langgraph / Tool Call Agent Test` → Tests LangGraph JS with tool calling + +## Supported Frameworks + +| Type | Platform | Framework | Streaming | Execution Modes | +| ------ | ---------- | --------------- | --------- | --------------- | +| llm | JavaScript | `openai` | both | - | +| llm | JavaScript | `anthropic` | both | - | +| llm | JavaScript | `google-genai` | both | - | +| llm | JavaScript | `langchain` | both | - | +| llm | Python | `openai` | both | sync/async | +| llm | Python | `anthropic` | both | sync/async | +| llm | Python | `langchain` | both | sync/async | +| llm | Python | `litellm` | both | sync/async | +| agents | JavaScript | `vercel` | - | - | +| agents | JavaScript | `langgraph` | - | - | +| agents | JavaScript | `mastra` | - | - | +| agents | Python | `openai-agents` | - | async | +| agents | Python | `langgraph` | - | sync/async | +| agents | Python | `pydantic-ai` | - | async | +| agents | Python | `google-genai` | - | sync/async | + +## Test Cases + +Test cases are defined in `src/test-cases/` and apply to frameworks based on their **type**. + +### LLM Test Cases (for `llm` type frameworks) + +| Test Case | Description | +| ---------------------- | ----------------------------------------- | +| `Basic LLM Test` | Single completion with system message | +| `Multi Turn LLM Test` | Multi-turn conversation | +| `Basic Error LLM Test` | API error handling | +| `Vision LLM Test` | Image input processing | +| `Long Input LLM Test` | Message trimming for large inputs (>20KB) | + +### Agent Test Cases (for `agents` type frameworks) + +| Test Case | Description | +| ----------------------- | --------------------------------------- | +| `Basic Agent Test` | Agent without tools (simple completion) | +| `Tool Call Agent Test` | Agent with successful tool calling | +| `Tool Error Agent Test` | Agent with tool that raises exception | +| `Vision Agent Test` | Agent that processes images | +| `Long Input Agent Test` | Agent with large input trimming | + +## Check Functions + +Each test case specifies an explicit list of **checks** that validate the captured Sentry spans. Checks are reusable functions defined in `src/test-cases/checks.ts`. + +### Check Structure + +A check is an object with a `name` and validation function: + +```typescript +interface Check { + name: string; + fn: ( + spans: CapturedSpan[], + config: FrameworkConfig, + testDef: TestDefinition, + ) => void; +} +``` + +Test cases explicitly list their checks: + +```typescript +export const basicLLMTest: TestDefinition = { + name: "Basic LLM Test", + type: "llm", + inputs: [...], + + checks: [ + checkAISpanCount(1), + checkChatSpanAttributes, + checkValidTokenUsage, + checkInputTokensCached, + checkOutputTokensReasoning, + ], +}; +``` + +### Available Checks -# All Python SDKs -npm run cli run py +#### Structure Checks -# All SDKs matching "lang" (langchain + langgraph in both JS and Python) -npm run cli run lang +| Check | Description | +| --------------------- | ------------------------------------------ | +| `checkAISpanCount(n)` | Factory function to validate AI span count | -# Specific SDK name across languages (js/langchain + py/langchain) -npm run cli run langchain +The `checkAISpanCount` factory function accepts: -# Specific SDK with exact path -npm run cli run js/langgraph +- A number for exact count: `checkAISpanCount(1)`, `checkAISpanCount(3)` +- An object with bounds: `checkAISpanCount({ min: 1 })`, `checkAISpanCount({ max: 5 })`, `checkAISpanCount({ min: 2, max: 4 })` -# SDK that only exists in one language -npm run cli run pydantic-ai +#### Span Type Attribute Checks -# Specific test case across all SDKs -npm run cli run -- --case 1-simple +| Check | Description | +| ------------------------------- | ----------------------------------------------------------------- | +| `checkChatSpanAttributes` | Validates chat/completion spans (model, messages, tokens) | +| `checkAgentSpanAttributes` | Validates agent invocation spans (gen_ai.agent.name) | +| `checkToolSpanAttributes` | Validates tool execution spans (type, name, description) | +| `checkHandoffSpanAttributes` | Validates handoff spans (agent-to-agent transfers) | +| `checkAvailableTools` | Validates gen_ai.request.available_tools matches test's tool defs | +| `checkResponseToolCalls([...])` | Factory to validate gen_ai.response.tool_calls on chat spans | +| `checkToolCalls([...])` | Factory to validate tool execution spans with input/output | -# Combine filters (langchain SDKs running 1-simple test only) -npm run cli run langchain -- --case 1-simple +Each check **fails if no spans of that type are found**. Use these to verify the expected span types are captured. + +**Tool validation factories:** + +```typescript +// Validate tool calls in LLM response (gen_ai.response.tool_calls) +checkResponseToolCalls([ + { name: "add", arguments: { a: 3, b: 5 } }, + { name: "multiply", arguments: { a: 8, b: 4 } }, +]); + +// Validate tool execution spans (gen_ai.tool.*) +checkToolCalls([ + { + name: "add", + type: "function", + description: "Add two numbers together", + input: { a: 3, b: 5 }, + output: 8, + }, +]); ``` -### CLI Filter Syntax +#### Token Checks + +| Check | Description | +| ---------------------------- | ------------------------------------------------------- | +| `checkValidTokenUsage` | Token counts exist on invoke_agent and chat spans | +| `checkInputTokensCached` | Cached tokens ≤ input tokens (skips if not present) | +| `checkOutputTokensReasoning` | Reasoning tokens ≤ output tokens (skips if not present) | + +#### Message Schema Checks + +| Check | Description | +| -------------------------- | ------------------------------------------------------------ | +| `checkInputMessagesSchema` | Validates `gen_ai.input.messages` follows Sentry conventions | + +The `checkInputMessagesSchema` check validates that the input messages attribute follows the [Sentry conventions schema](https://getsentry.github.io/sentry-conventions/generated/attributes/gen_ai.html#gen_aiinputmessages): + +- Must be an array of message objects +- Each message must have a `role` field: "user", "assistant", "tool", or "system" +- Each message must have a `parts` array (new format) or `content` field (legacy) +- Parts can have types: "text", "tool_call", "tool_call_response", "image" +- Validates type-specific fields (e.g., tool_call must have name) + +#### Message Trimming Checks + +| Check | Description | +| ----------------------- | ----------------------------------- | +| `checkMessageTrimming` | Messages are trimmed below 15KB | +| `checkTrimmingMetadata` | Original length metadata is present | + +#### Agent-specific Checks + +| Check | Description | +| --------------------- | --------------------------------------------------------------------------------- | +| `checkAgentHierarchy` | Validates agent span hierarchy and `gen_ai.agent.name` propagation to child spans | + +### Checks by Test Case + +#### LLM Tests + +**Basic LLM Test:** + +- `checkAISpanCount(1)`, `checkChatSpanAttributes`, `checkValidTokenUsage`, `checkInputMessagesSchema`, `checkInputTokensCached`, `checkOutputTokensReasoning` + +**Multi-Turn LLM Test:** + +- `checkAISpanCount(3)`, `checkChatSpanAttributes`, `checkValidTokenUsage`, `checkTokenProgression` (inline), `checkInputMessagesSchema`, `checkInputTokensCached`, `checkOutputTokensReasoning` + +**Basic Error LLM Test:** + +- `checkAISpanCount({ min: 1 })`, `checkErrorCaptured` (inline) + +**Vision LLM Test:** + +- `checkChatSpanAttributes`, `checkValidTokenUsage`, `checkInputMessagesSchema` -The CLI supports flexible filtering to run exactly the tests you need: +**Long Input LLM Test:** -**Filter Types:** +- `checkChatSpanAttributes`, `checkMessageTrimming`, `checkTrimmingMetadata`, `checkInputMessagesSchema` -- **Language filter**: `js` or `py` - Runs all SDKs in that language -- **Exact path**: `js/openai` - Runs a specific SDK -- **Partial name match**: Any string that matches SDK names (uses `contains`) - - `lang` → matches `langchain`, `langgraph` (in both JS and Python) - - `langchain` → matches only `langchain` (in both JS and Python) - - `pydantic` → matches only `pydantic-ai` (Python only) - - `openai` → matches `openai`, `openai-agents` (in all languages) +#### Agent Tests -**Additional Options:** +**Basic Agent Test:** -- `--case ` - Filter to specific test case (e.g., `1-simple`) -- `--all` - Run all tests across all SDKs -- `--verbose` - Show detailed output including LLM responses -- `--reports ` - Generate reports (ctrf, html, or all) +- `checkAgentSpanAttributes`, `checkChatSpanAttributes`, `checkValidTokenUsage`, `checkAgentHierarchy`, `checkInputMessagesSchema`, `checkInputTokensCached`, `checkOutputTokensReasoning` -**Examples:** +**Tool Call Agent Test:** + +- `checkAgentSpanAttributes`, `checkChatSpanAttributes`, `checkToolSpanAttributes`, `checkValidTokenUsage`, `checkAgentHierarchy`, `checkAvailableTools`, `checkResponseToolCalls([...])`, `checkToolCalls([...])`, `checkInputMessagesSchema`, `checkInputTokensCached`, `checkOutputTokensReasoning` + +**Tool Error Agent Test:** + +- `checkAgentSpanAttributes`, `checkChatSpanAttributes`, `checkToolSpanAttributes`, `checkAgentHierarchy`, `checkAvailableTools`, `checkResponseToolCalls([...])`, `checkInputMessagesSchema`, `checkToolErrorSpan` (inline) + +**Vision Agent Test:** + +- `checkAgentSpanAttributes`, `checkChatSpanAttributes`, `checkValidTokenUsage`, `checkAgentHierarchy`, `checkInputMessagesSchema` + +**Long Input Agent Test:** + +- `checkAgentSpanAttributes`, `checkChatSpanAttributes`, `checkMessageTrimming`, `checkTrimmingMetadata`, `checkAgentHierarchy`, `checkInputMessagesSchema` + +## How It Works + +1. **Discovery**: Scans `templates/` directory for framework configurations +2. **Matrix Generation**: Creates test matrix (framework × test × execution modes) +3. **Template Rendering**: Generates runnable test files using Nunjucks templates +4. **Execution**: Runs tests with Sentry DSN pointing to local span collector +5. **Validation**: Runs each check function against captured spans +6. **Reporting**: Generates console output and CTRF JSON report + +``` +TestDefinition (TypeScript) + Framework Template (Nunjucks) + ↓ + Template Renderer generates test file + ↓ + Runner executes test file + ↓ + Sentry SDK sends spans to Span Collector + ↓ + Validator runs checks array on captured spans + ↓ + Reporter outputs results +``` + +## Adding a New Framework + +### 1. Create Template Directory + +```bash +mkdir -p src/runner/templates/{llm|agents}/{js|py}/your-framework +``` + +### 2. Create `config.json` + +```json +{ + "name": "your-framework", + "displayName": "Your Framework SDK", + "type": "llm-only", + "platform": "js", + "streamingMode": "both", + "dependencies": [{ "package": "your-framework", "version": "framework" }], + "versions": ["1.0.0"], + "sentryVersions": ["latest"] +} +``` + +### 3. Create `template.njk` + +```njk +{% extends "base.js.njk" %} + +{% block setup %} +let client; +{% endblock %} + +{% block dynamic_imports %} + const SDK = (await import("your-framework")).default; + client = new SDK(); +{% endblock %} + +{% block test %} +{% for input in inputs %} + const response = await client.complete({ + model: "{{ input.model }}", + messages: {{ input.messages | dump }}, + }); + console.log("Response:", response.content); +{% endfor %} +{% endblock %} +``` + +### 4. Build and Test ```bash -# Quick language-wide tests -npm run cli run js # All JS SDKs -npm run cli run py # All Python SDKs +npm run build +npm run test -- --framework your-framework --verbose +``` -# Partial matching for related SDKs -npm run cli run lang # langchain + langgraph (both languages) -npm run cli run openai # openai + openai-agents (all languages) +## Adding a New Test Case + +### 1. Create Test File + +Test cases use an explicit `checks` array to define validations: + +```typescript +// src/test-cases/llm/your-test.ts +import { TestDefinition } from "../../types.js"; +import { + checkAISpanCount, + checkChatSpanAttributes, + checkValidTokenUsage, +} from "../checks.js"; + +export const yourTest: TestDefinition = { + name: "Your Test Name", + description: "What this test validates", + type: "llm", // or 'agent' + + inputs: [ + { + model: "gpt-4o-mini", + messages: [{ role: "user", content: "Test prompt" }], + }, + ], + + checks: [ + checkAISpanCount({ min: 1 }), + checkChatSpanAttributes, + checkValidTokenUsage, + ], +}; + +export default yourTest; +``` + +### 2. Adding Custom Checks + +For test-specific validations, define custom checks inline: + +```typescript +import { expect } from "chai"; +import { TestDefinition, Check } from "../../types.js"; +import { checkAISpanCount } from "../checks.js"; +import { extractGenAISpans, skipIf } from "../utils.js"; + +// Custom check for this test +const checkSpecificBehavior: Check = { + name: "checkSpecificBehavior", + fn: (spans, config, testDef) => { + const aiSpans = extractGenAISpans(spans); + skipIf(aiSpans.length === 0, "No AI spans captured"); + + // Your custom validation logic + expect(aiSpans[0].data?.["custom.attribute"]).to.exist; + }, +}; + +export const yourTest: TestDefinition = { + name: "Your Test Name", + type: "llm", + inputs: [...], + + checks: [ + checkAISpanCount({ min: 1 }), + checkSpecificBehavior, // Custom check + ], +}; +``` -# Exact SDK selection -npm run cli run langchain # js/langchain + py/langchain -npm run cli run js/langgraph # Only js/langgraph +### 3. Register in Index -# Test case filtering -npm run cli run -- --case 1-simple # Run 1-simple across all SDKs -npm run cli run lang -- --case 1-simple # Run 1-simple on lang* SDKs +```typescript +// src/test-cases/index.ts +import { yourTest } from "./llm/your-test.js"; -# List available SDKs -npm run cli list +export const testCases = { + llm: { + // ... existing tests + yourTest: yourTest, + }, +}; +``` + +### 4. Build and Test + +```bash +npm run build +npm run test -- --test "Your Test Name" --verbose +``` + +## Framework Configuration + +Each framework has a `config.json` with these fields: + +| Field | Description | +| ---------------- | --------------------------------------------- | +| `name` | Framework identifier | +| `displayName` | Human-readable name | +| `type` | `"llm-only"` or `"agentic"` | +| `platform` | `"js"` or `"py"` | +| `streamingMode` | `"streaming"`, `"blocking"`, or `"both"` | +| `executionMode` | Python only: `"sync"`, `"async"`, or `"both"` | +| `dependencies` | NPM/pip packages to install | +| `versions` | Framework versions to test | +| `sentryVersions` | Sentry SDK versions to test against | +| `modelOverrides` | Override model names for validation | +| `skip` | Tests or checks to skip | + +## Test Utilities + +Available in `src/test-cases/utils.ts`: + +### Core Utilities + +| Function | Purpose | +| ---------------------- | -------------------------------------- | +| `skip(reason)` | Skip the current check with a reason | +| `skipIf(cond, reason)` | Conditionally skip a check | +| `extractGenAISpans()` | Filter spans for `gen_ai.*` operations | +| `checkTokenUsage()` | Validate token count attributes | +| `assertAttributes()` | Schema-based attribute validation | +| `printSpanSummary()` | Debug helper to print captured spans | + +### Span Type Filters + +| Function | Purpose | +| -------------------- | -------------------------------------------- | +| `findAgentSpans()` | Find `invoke_agent` spans (top-level agents) | +| `findChatSpans()` | Find `chat`/`completion` spans (LLM calls) | +| `findToolSpans()` | Find tool execution spans | +| `findHandoffSpans()` | Find agent-to-agent handoff spans | + +### Tool Input Validation + +| Function | Purpose | +| ------------------- | -------------------------------------------- | +| `assertToolInput()` | Validate tool input arguments against schema | +| `getToolInput()` | Get parsed tool input arguments from span | + +### Attribute Schema + +The `assertAttributes` function supports flexible matching: + +```typescript +assertAttributes(spans, { + "gen_ai.operation.name": true, // Must exist (any value) + "gen_ai.request.model": "gpt-4", // Exact match + "gen_ai.response.model": "gpt-4*", // Pattern match (wildcard) + sensitive_field: false, // Must NOT exist +}); +``` + +### Tool Input Schema + +The `assertToolInput` function validates tool arguments: + +```typescript +const toolSpans = findToolSpans(extractGenAISpans(spans)); +assertToolInput(toolSpans[0], { + a: true, // Argument must exist + b: true, // Argument must exist + optional: false, // Argument must NOT exist +}); +``` + +## Environment Variables + +Create a `.env` file with your API keys: + +```bash +OPENAI_API_KEY=sk-... +ANTHROPIC_API_KEY=sk-ant-... +GOOGLE_API_KEY=... +``` + +## Debugging + +### Verbose Mode + +```bash +npm run test -- --framework openai --verbose +``` + +### Setup Only (Inspect Generated Files) + +```bash +npm run test setup -- --framework openai +# Check runs/ directory for generated test files +``` + +### Print Span Data + +Use `printSpanSummary()` in a custom check: + +```typescript +import { printSpanSummary } from "../utils.js"; + +const debugCheck: Check = { + name: "debugCheck", + fn: (spans) => { + printSpanSummary(spans); + }, +}; ``` ## Using as a GitHub Action -This repository can be used as a reusable GitHub Action in SDK repositories (e.g., `sentry-javascript`, `sentry-python`) to run AI integration tests on a schedule. +This repository can be used as a reusable GitHub Action in SDK repositories to run AI integration tests on a schedule. ### Setup in SDK Repositories -1. **Create a workflow** in your SDK repo (e.g., `.github/workflows/ai-integration-tests.yml`): +Create a workflow in your SDK repo (e.g., `.github/workflows/ai-integration-tests.yml`): ```yaml name: AI Integration Tests @@ -197,7 +731,7 @@ name: AI Integration Tests on: schedule: - cron: "0 9 * * 1" # Weekly on Monday at 9am UTC - workflow_dispatch: # Allow manual trigger + workflow_dispatch: jobs: test: @@ -209,30 +743,72 @@ jobs: - name: Run AI Integration Tests uses: getsentry/testing-ai-sdk-integrations@v1 with: - language: js # or 'python' + platform: py # or 'js', or leave empty for both openai-api-key: ${{ secrets.OPENAI_API_KEY }} anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} google-api-key: ${{ secrets.GOOGLE_API_KEY }} ``` -2. **Add secrets** to your SDK repository: - - `OPENAI_API_KEY` - OpenAI API key - - `ANTHROPIC_API_KEY` - Anthropic API key - - `GOOGLE_API_KEY` - Google API key for GenAI - - `GITHUB_TOKEN` - GitHub token +### Action Inputs + +| Input | Required | Default | Description | +| ------------------------ | -------- | ------------- | -------------------------------------------------------- | +| `platform` | No | `""` | Platform to test: `js`, `py`, or empty for both | +| `framework` | No | `""` | Specific framework to test (e.g., `openai`, `langchain`) | +| `test` | No | `""` | Specific test to run (e.g., `Basic LLM Test`) | +| `parallel` | No | `4` | Number of tests to run in parallel | +| `sentry-python-path` | No | `""` | Path to local sentry-python for editable install | +| `sentry-javascript-path` | No | `""` | Path to local sentry-javascript for linking | +| `openai-api-key` | Yes | - | OpenAI API key | +| `anthropic-api-key` | Yes | - | Anthropic API key | +| `google-api-key` | Yes | - | Google API key for GenAI | +| `google-vertex-project` | No | `""` | Google Vertex AI project ID | +| `google-vertex-location` | No | `us-central1` | Google Vertex AI location | + +### Action Outputs + +| Output | Description | +| --------- | --------------------------------------------- | +| `success` | `true` if all tests passed, `false` otherwise | +| `total` | Total number of tests run | +| `passed` | Number of tests that passed | +| `failed` | Number of tests that failed | + +### Advanced Usage + +```yaml +# Test specific framework with local SDK +- name: Run AI Integration Tests + id: ai-tests + uses: getsentry/testing-ai-sdk-integrations@v1 + with: + platform: py + framework: openai + parallel: 8 + sentry-python-path: ${{ github.workspace }} + openai-api-key: ${{ secrets.OPENAI_API_KEY }} + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + google-api-key: ${{ secrets.GOOGLE_API_KEY }} + +- name: Check results + run: | + echo "Success: ${{ steps.ai-tests.outputs.success }}" + echo "Passed: ${{ steps.ai-tests.outputs.passed }}/${{ steps.ai-tests.outputs.total }}" +``` ### How It Works -- The action runs tests for the specified language (js or python) -- On failure, it automatically creates or updates an issue in the **calling repository** (not this repo) +- The action installs dependencies and builds the test framework +- Runs tests for the specified platform/framework with parallel execution +- Uploads test results as artifacts +- On failure, automatically creates or updates an issue in the calling repository - Issues are labeled with `ai-integration-test-failure` for easy tracking -- Test results are included in the issue body in JSON format - -## Test Scenarios +- Test results with detailed failure information are included in the issue body -Each SDK implementation includes these scenarios (where supported by the SDK): +## References -1. **Simple Chat** - Basic request-response completion -2. **Streaming** - Streaming response handling -3. **Function Calling** - Tool/function calling capabilities -4. **Error Handling** - Application errors and invalid inputs +- **Sentry JavaScript SDK:** https://github.com/getsentry/sentry-javascript +- **Sentry Python SDK:** https://github.com/getsentry/sentry-python +- **Vercel AI SDK:** https://sdk.vercel.ai/docs +- **OpenAI Python SDK:** https://github.com/openai/openai-python +- **Mastra AI Framework:** https://mastra.ai/docs diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..f7dbcda --- /dev/null +++ b/TESTING.md @@ -0,0 +1,131 @@ +# Testing Guide + +## Testing Template Rendering + +To test framework template rendering without running the full orchestrator: + +```bash +npm run test-templates +``` + +This will: +1. Render both OpenAI SDK (LLM) and OpenAI Agents (agentic) templates +2. Show the rendered Python code +3. Save output files for inspection +4. Display framework configurations +5. Calculate test matrix combinations + +**Output files:** +- `test-output-openai-llm.py` - Rendered LLM test +- `test-output-openai-agents.py` - Rendered agent test + +**Cleanup:** +```bash +rm test-output-*.py +``` + +## Manual Template Testing + +You can also test templates programmatically: + +```javascript +import { TemplateRenderer } from './dist/runner/template-renderer.js'; + +const renderer = new TemplateRenderer(); + +// Render a template directly +const code = renderer.render('llm/py/openai/template.njk', { + testName: 'My Test', + frameworkName: 'openai', + system: 'You are a helpful assistant.', + input: { + model: 'gpt-4o', + prompt: 'Hello!' + } +}); + +// Or use the helper method +const code2 = renderer.renderFramework('llm', 'py', 'openai', { + testName: 'My Test', + frameworkName: 'openai', + system: 'You are a helpful assistant.', + input: { + model: 'gpt-4o', + prompt: 'Hello!' + } +}); + +console.log(code); +``` + +## Validating Python Syntax + +Generated Python files can be validated: + +```bash +python3 -m py_compile test-output-openai-llm.py +``` + +## Test Matrix + +Framework configurations define test matrices: + +```json +{ + "versions": ["1.57.0", "1.58.1"], + "sentryVersions": ["2.19.2", "latest"] +} +``` + +This generates 4 test combinations (2 × 2): +- openai 1.57.0 + sentry-sdk 2.19.2 +- openai 1.57.0 + sentry-sdk latest +- openai 1.58.1 + sentry-sdk 2.19.2 +- openai 1.58.1 + sentry-sdk latest + +## Framework Configuration + +Each framework has its own directory with two files: +- `template.njk` - Nunjucks template +- `config.json` - Framework configuration + +Example: +- `src/runner/templates/llm/py/openai/template.njk` +- `src/runner/templates/llm/py/openai/config.json` + +Schema: +```typescript +{ + name: string; // Framework identifier + displayName: string; // Human-readable name + type: 'llm-only' | 'agentic'; + platform: 'js' | 'py'; + dependencies: Array<{ + package: string; + version: string; // "framework" | "latest" | specific version + }>; + versions: string[]; // Framework versions to test + sentryVersions: string[]; // Sentry SDK versions + matrix?: { // Optional additional axes + modelProviders?: string[]; + [key: string]: string[]; + }; +} +``` + +## Adding New Framework Templates + +1. Create framework directory: `src/runner/templates/{llm,agents}/{js,py}/{framework}/` +2. Create template file: `template.njk` +3. Create config file: `config.json` +4. Rebuild: `npm run build` +5. Test: `npm run test-templates` + +Example for a new framework called "my-sdk": +```bash +mkdir -p src/runner/templates/llm/py/my-sdk +touch src/runner/templates/llm/py/my-sdk/template.njk +touch src/runner/templates/llm/py/my-sdk/config.json +``` + +See template documentation in `src/runner/templates/README.md`. diff --git a/action.yml b/action.yml index 79de77e..77c4479 100644 --- a/action.yml +++ b/action.yml @@ -7,14 +7,30 @@ branding: color: "purple" inputs: - language: - description: "SDK language to test. One of: js, py, or leave empty for both" + platform: + description: "Platform to test. One of: js, py, or leave empty for both" required: false default: "" + framework: + description: "Specific framework to test (e.g., openai, anthropic, langchain)" + required: false + default: "" + test: + description: "Specific test to run (e.g., 'Basic LLM Test')" + required: false + default: "" + parallel: + description: "Number of tests to run in parallel" + required: false + default: "4" sentry-python-path: - description: "Path to local sentry-python repository" + description: "Path to local sentry-python repository for editable install" required: false - default: "../../../sentry-python/sentry-python" + default: "" + sentry-javascript-path: + description: "Path to local sentry-javascript repository for linking" + required: false + default: "" openai-api-key: description: "OpenAI API key" required: true @@ -24,11 +40,28 @@ inputs: google-api-key: description: "Google API key for GenAI" required: true + google-vertex-project: + description: "Google Vertex AI project ID" + required: false + default: "" + google-vertex-location: + description: "Google Vertex AI location" + required: false + default: "us-central1" outputs: success: description: "Whether all tests passed" value: ${{ steps.run-tests.outputs.success }} + total: + description: "Total number of tests run" + value: ${{ steps.run-tests.outputs.total }} + passed: + description: "Number of tests passed" + value: ${{ steps.run-tests.outputs.passed }} + failed: + description: "Number of tests failed" + value: ${{ steps.run-tests.outputs.failed }} runs: using: "composite" @@ -39,27 +72,21 @@ runs: node-version: "20" - name: Setup Python - if: inputs.language == 'py' || inputs.language == '' + if: inputs.platform == 'py' || inputs.platform == '' uses: actions/setup-python@v5 with: - python-version: "3.13" + python-version: "3.12" - - name: Install orchestration dependencies - shell: bash - working-directory: ${{ github.action_path }}/shared/orchestration - run: npm install + - name: Install uv (Python package manager) + if: inputs.platform == 'py' || inputs.platform == '' + uses: astral-sh/setup-uv@v5 - - name: Setup SDK dependencies + - name: Install dependencies and build shell: bash working-directory: ${{ github.action_path }} run: | - if [ "${{ inputs.language }}" = "py" ]; then - npm run cli setup -- --local-sentry-python ${{ inputs.sentry-python-path }} - elif [ -z "${{ inputs.language }}" ]; then - npm run cli setup - else - npm run cli setup ${{ inputs.language }} - fi + npm install + npm run build - name: Run tests id: run-tests @@ -69,16 +96,62 @@ runs: OPENAI_API_KEY: ${{ inputs.openai-api-key }} ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }} GOOGLE_API_KEY: ${{ inputs.google-api-key }} + GOOGLE_VERTEX_PROJECT: ${{ inputs.google-vertex-project }} + GOOGLE_VERTEX_LOCATION: ${{ inputs.google-vertex-location }} run: | set +e - if [ -z "${{ inputs.language }}" ]; then - npm run cli run -- --all --reports ctrf - else - npm run cli run ${{ inputs.language }} -- --reports ctrf + + # Build command arguments + ARGS="" + + if [ -n "${{ inputs.platform }}" ]; then + ARGS="$ARGS --platform ${{ inputs.platform }}" fi + + if [ -n "${{ inputs.framework }}" ]; then + ARGS="$ARGS --framework ${{ inputs.framework }}" + fi + + if [ -n "${{ inputs.test }}" ]; then + ARGS="$ARGS --test \"${{ inputs.test }}\"" + fi + + if [ -n "${{ inputs.parallel }}" ]; then + ARGS="$ARGS -j=${{ inputs.parallel }}" + fi + + if [ -n "${{ inputs.sentry-python-path }}" ]; then + ARGS="$ARGS --sentry-python ${{ inputs.sentry-python-path }}" + fi + + if [ -n "${{ inputs.sentry-javascript-path }}" ]; then + ARGS="$ARGS --sentry-javascript ${{ inputs.sentry-javascript-path }}" + fi + + # Run tests + echo "Running: npm run test -- $ARGS" + eval "npm run test -- $ARGS" EXIT_CODE=$? set -e + # Parse CTRF report for outputs (find latest report with timestamp) + CTRF_REPORT=$(ls -t test-results/ctrf-report-*.json 2>/dev/null | head -1) + if [ -n "$CTRF_REPORT" ] && [ -f "$CTRF_REPORT" ]; then + TOTAL=$(jq -r '.results.summary.tests' "$CTRF_REPORT") + PASSED=$(jq -r '.results.summary.passed' "$CTRF_REPORT") + FAILED=$(jq -r '.results.summary.failed' "$CTRF_REPORT") + + echo "total=$TOTAL" >> $GITHUB_OUTPUT + echo "passed=$PASSED" >> $GITHUB_OUTPUT + echo "failed=$FAILED" >> $GITHUB_OUTPUT + echo "ctrf_report=$CTRF_REPORT" >> $GITHUB_OUTPUT + else + echo "total=0" >> $GITHUB_OUTPUT + echo "passed=0" >> $GITHUB_OUTPUT + echo "failed=0" >> $GITHUB_OUTPUT + echo "ctrf_report=" >> $GITHUB_OUTPUT + fi + if [ $EXIT_CODE -eq 0 ]; then echo "success=true" >> $GITHUB_OUTPUT else @@ -87,6 +160,14 @@ runs: exit $EXIT_CODE + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results + path: ${{ github.action_path }}/test-results/ + if-no-files-found: ignore + - name: Create or update issue on failure if: failure() uses: actions/github-script@v7 @@ -95,16 +176,25 @@ runs: const fs = require('fs'); const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - // Helper functions const formatDuration = (ms) => ms < 1000 ? `${ms}ms` : `${(ms / 1000).toFixed(2)}s`; const getStatusIcon = (status) => { - const icons = { passed: '✓', failed: '✗', skipped: '○' }; + const icons = { passed: '\u2713', failed: '\u2717', skipped: '\u25CB' }; return icons[status] || '-'; }; - // Try to read and format test results from CTRF report let resultsContent = 'Test execution failed. Check the [workflow run](' + runUrl + ') for details.'; - const ctrfPath = '${{ github.action_path }}/shared/orchestration/test-results/ctrf-report.json'; + // Find latest CTRF report + const testResultsDir = '${{ github.action_path }}/test-results'; + let ctrfPath = ''; + if (fs.existsSync(testResultsDir)) { + const files = fs.readdirSync(testResultsDir) + .filter(f => f.startsWith('ctrf-report-') && f.endsWith('.json')) + .sort() + .reverse(); + if (files.length > 0) { + ctrfPath = testResultsDir + '/' + files[0]; + } + } if (fs.existsSync(ctrfPath)) { try { @@ -113,47 +203,49 @@ runs: const tests = report.results.tests; const duration = summary.stop - summary.start; - // Build summary table - let content = '## 📊 Summary\n\n'; + let content = '## Summary\n\n'; content += '| Metric | Value |\n|--------|-------|\n'; content += `| **Total Tests** | ${summary.tests} |\n`; - content += `| **✓ Passed** | ${summary.passed} |\n`; - content += `| **✗ Failed** | ${summary.failed} |\n`; + content += `| **Passed** | ${summary.passed} |\n`; + content += `| **Failed** | ${summary.failed} |\n`; + content += `| **Skipped** | ${summary.skipped || 0} |\n`; content += `| **Duration** | ${formatDuration(duration)} |\n\n`; - // Build test matrix - const sdks = [...new Set(tests.map(t => (t.suite?.[0] || 'unknown')))].sort(); - const testCases = [...new Set(tests.map(t => t.name.split(' :: ')[1] || t.name))].sort(); - - const testMap = new Map(); + const byFramework = {}; tests.forEach(test => { - const caseId = test.name.split(' :: ')[1] || test.name; - const suite = test.suite?.[0] || 'unknown'; - testMap.set(`${suite}::${caseId}`, test); + const framework = test.suite && test.suite[0] ? test.suite[0] : 'unknown'; + if (!byFramework[framework]) byFramework[framework] = []; + byFramework[framework].push(test); }); - content += '## 🧪 Test Matrix\n\n'; - content += '| SDK | ' + testCases.join(' | ') + ' |\n'; - content += '|-----|' + testCases.map(() => '-----').join('|') + '|\n'; - - sdks.forEach(sdk => { - content += `| **${sdk}** |`; - testCases.forEach(caseId => { - const test = testMap.get(`${sdk}::${caseId}`); - content += test ? ` ${getStatusIcon(test.status)} |` : ' - |'; + content += '## Results by Framework\n\n'; + + Object.keys(byFramework).sort().forEach(framework => { + const frameworkTests = byFramework[framework]; + const failed = frameworkTests.filter(t => t.status === 'failed').length; + + const statusIcon = failed > 0 ? 'FAILED' : 'PASSED'; + content += `### ${statusIcon} - ${framework}\n\n`; + content += '| Test | Status | Duration |\n|------|--------|----------|\n'; + + frameworkTests.forEach(test => { + const testName = test.name.replace(framework + ' :: ', ''); + const icon = getStatusIcon(test.status); + const dur = test.duration ? formatDuration(test.duration) : '-'; + content += `| ${testName} | ${icon} ${test.status} | ${dur} |\n`; }); content += '\n'; }); - content += '\n'; - // Build failed tests details const failedTests = tests.filter(t => t.status === 'failed'); if (failedTests.length > 0) { - content += '## ❌ Failed Tests Details\n\n'; + content += '## Failed Tests Details\n\n'; failedTests.forEach(test => { - const caseId = test.name.split(' :: ')[1] || test.name; - const suite = test.suite?.[0] || 'unknown'; - content += `
\n${suite} :: ${caseId} (${test.duration}ms)\n\n`; + const framework = test.suite && test.suite[0] ? test.suite[0] : 'unknown'; + const testName = test.name.replace(framework + ' :: ', ''); + content += '
\n'; + content += `${framework} - ${testName}\n\n`; + if (test.message) content += `**Error:** ${test.message}\n\n`; if (test.trace) content += '```\n' + test.trace + '\n```\n'; content += '
\n\n'; }); @@ -167,27 +259,25 @@ runs: const issueLabel = 'ai-integration-test-failure'; const date = new Date(); - const formattedDate = date.toLocaleString('en-US', { - year: 'numeric', - month: 'long', - day: 'numeric', - hour: '2-digit', - minute: '2-digit', - timeZone: 'UTC', - timeZoneName: 'short' - }); - const body = `## AI Integration Test Failure - - **Date**: ${formattedDate} - **Workflow Run**: ${runUrl} + const formattedDate = date.toISOString(); - ${resultsContent} + const platform = '${{ inputs.platform }}' || 'all'; + const framework = '${{ inputs.framework }}' || 'all'; - --- - *This issue was automatically created by the [AI Integration Testing framework](https://github.com/getsentry/testing-ai-sdk-integrations).* - `; + const body = [ + '## AI Integration Test Failure', + '', + `**Date**: ${formattedDate}`, + `**Platform**: ${platform}`, + `**Framework**: ${framework}`, + `**Workflow Run**: ${runUrl}`, + '', + resultsContent, + '', + '---', + '*This issue was automatically created by the [AI Integration Testing framework](https://github.com/getsentry/testing-ai-sdk-integrations).*' + ].join('\n'); - // Check for existing open issues with the label const issues = await github.rest.issues.listForRepo({ owner: context.repo.owner, repo: context.repo.repo, @@ -196,7 +286,6 @@ runs: }); if (issues.data.length > 0) { - // Update existing issue await github.rest.issues.update({ owner: context.repo.owner, repo: context.repo.repo, @@ -205,11 +294,10 @@ runs: }); console.log(`Updated existing issue #${issues.data[0].number}`); } else { - // Create new issue await github.rest.issues.create({ owner: context.repo.owner, repo: context.repo.repo, - title: '❌ AI Integration Tests Failed', + title: 'AI Integration Tests Failed', body: body, labels: [issueLabel, 'automated'] }); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..15c7323 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,452 @@ +# Architecture + +## Overview + +Test framework for Sentry AI SDK integrations. Test definitions (TypeScript) combined with framework templates (Nunjucks) generate runnable test files. A span collector HTTP server captures Sentry data for validation. + +## Components + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Test Definition │ +│ { name, type, inputs, agent?, checks: Check[] } │ +└─────────────────────────────────────────────────────────────────┘ + │ + ┌─────────────────┴─────────────────┐ + │ static config │ checks array + ▼ ▼ +┌───────────────────────┐ ┌───────────────────────┐ +│ Runner │ │ Validator │ +│ + Framework Template │ │ (Chai assertions) │ +└───────────────────────┘ └───────────────────────┘ + │ ▲ + │ rendered test │ spans + ▼ │ +┌───────────────────────┐ ┌───────────────────────┐ +│ Test Execution │─────────────▶│ Span Collector │ +│ (runs/ directory) │ Sentry │ (HTTP server) │ +└───────────────────────┘ └───────────────────────┘ +``` + +### Orchestrator (`src/orchestrator.ts`) +Entry point. Discovers frameworks, builds test matrix, coordinates execution, generates reports. + +### Span Collector (`src/span-collector/`) +HTTP server that mimics Sentry's envelope endpoint. Creates dynamic DSN endpoints per test run, collects spans. + +### Test Cases (`src/test-cases/`) +TypeScript test definitions shared across all frameworks. Each test has a `type` ("llm" or "agent") that determines which frameworks can run it. + +### Runner (`src/runner/`) +- `runner.ts` - Main runner orchestration +- `javascript-runner.ts` - Node.js environment setup and execution +- `python-runner.ts` - Python/uv environment setup and execution +- `template-renderer.ts` - Nunjucks template rendering +- `framework-discovery.ts` - Auto-discovers frameworks from templates directory + +### Validator (`src/validator.ts`) +Runs each check function from the test definition's `checks` array against captured spans. + +### Reporters (`src/reporters/`) +- `ctrf-reporter.ts` - CTRF JSON report generation +- `live-status.ts` - Real-time terminal status display + +## Test Definition Format + +Test definitions use an explicit `checks` array with reusable check functions: + +```typescript +// src/test-cases/llm/basic.ts +import { TestDefinition } from "../../types.js"; +import { + checkAISpanCount, + checkChatSpanAttributes, + checkValidTokenUsage, + checkInputMessagesSchema, +} from "../checks.js"; + +export const basicLLMTest: TestDefinition = { + name: "Basic LLM Test", + description: "Single completion call with system message", + type: "llm", + + inputs: [ + { + model: "gpt-4o-mini", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "What is the capital of France?" }, + ], + }, + ], + + checks: [ + checkAISpanCount(1), + checkChatSpanAttributes, + checkValidTokenUsage, + checkInputMessagesSchema, + ], +}; +``` + +### Agent Test Definition + +```typescript +// src/test-cases/agents/tool-call.ts +import { TestDefinition } from "../../types.js"; +import { + checkAgentSpanAttributes, + checkChatSpanAttributes, + checkToolSpanAttributes, + checkValidTokenUsage, + checkToolCalls, +} from "../checks.js"; + +export const toolCallAgentTest: TestDefinition = { + name: "Tool Call Agent Test", + description: "Agent with successful tool calling", + type: "agent", + + agent: { + name: "math_assistant", + description: "A math assistant that can perform calculations", + tools: [ + { + name: "add", + description: "Add two numbers together", + parameters: { + type: "object", + properties: { + a: { type: "number", description: "First number" }, + b: { type: "number", description: "Second number" }, + }, + required: ["a", "b"], + }, + result: 8, + }, + ], + }, + + inputs: [ + { + model: "gpt-4o-mini", + messages: [ + { role: "user", content: "What is 3 + 5? Use the add tool." }, + ], + }, + ], + + checks: [ + checkAgentSpanAttributes, + checkChatSpanAttributes, + checkToolSpanAttributes, + checkValidTokenUsage, + checkToolCalls([{ name: "add", input: { a: 3, b: 5 }, output: 8 }]), + ], +}; +``` + +## Framework Classification + +| Type | Test Type | Supports | Examples | +|------|-----------|----------|----------| +| **llm-only** | `llm` | Simple completions | OpenAI SDK, Anthropic SDK, LangChain | +| **agentic** | `agent` | Agents with tools | Vercel AI, LangGraph, Mastra, OpenAI Agents | + +Frameworks with type `agentic` run agent tests. Frameworks with type `llm-only` run LLM tests. + +## Framework Templates + +Each framework has a directory with `config.json` and `template.njk`: + +``` +src/runner/templates/ +├── base.js.njk # Base JavaScript template +├── base.py.njk # Base Python template +├── llm/ # LLM-only frameworks +│ ├── js/ +│ │ ├── openai/ +│ │ │ ├── config.json +│ │ │ └── template.njk +│ │ ├── anthropic/ +│ │ ├── google-genai/ +│ │ └── langchain/ +│ └── py/ +│ ├── openai/ +│ ├── anthropic/ +│ ├── langchain/ +│ └── litellm/ +└── agents/ # Agentic frameworks + ├── js/ + │ ├── vercel/ + │ ├── langgraph/ + │ └── mastra/ + └── py/ + ├── openai-agents/ + ├── langgraph/ + ├── pydantic-ai/ + └── google-genai/ +``` + +### Framework Configuration (`config.json`) + +```json +{ + "name": "openai", + "displayName": "OpenAI JavaScript SDK", + "type": "llm-only", + "platform": "js", + "streamingMode": "both", + "dependencies": [ + { "package": "openai", "version": "framework" } + ], + "versions": ["4.96.0"], + "sentryVersions": ["latest"], + "modelOverrides": { + "request": "gpt-4o-mini", + "response": "gpt-4o-mini*" + }, + "skip": { + "tests": ["Long Input LLM Test"], + "checks": { + "Basic LLM Test": ["checkAgentHierarchy"] + } + } +} +``` + +### Configuration Fields + +| Field | Description | +|-------|-------------| +| `name` | Framework identifier | +| `displayName` | Human-readable name | +| `type` | `"llm-only"` or `"agentic"` | +| `platform` | `"js"` or `"py"` | +| `streamingMode` | `"streaming"`, `"blocking"`, or `"both"` | +| `executionMode` | Python only: `"sync"`, `"async"`, or `"both"` | +| `dependencies` | NPM/pip packages to install | +| `versions` | Framework versions to test | +| `sentryVersions` | Sentry SDK versions to test against | +| `modelOverrides` | Override model names for validation | +| `skip.tests` | Test names to skip entirely | +| `skip.checks` | Per-test check names to skip | + +## Directory Structure + +``` +testing-ai-sdk-integrations/ +├── package.json # Orchestrator dependencies +├── tsconfig.json +├── .env # API keys (gitignored) +│ +├── src/ # TypeScript source (ES modules) +│ ├── cli.ts # CLI entry point +│ ├── orchestrator.ts # Main test coordinator +│ ├── types.ts # Core type definitions +│ ├── validator.ts # Test validation logic +│ ├── setup.ts # Setup utilities +│ ├── concurrency.ts # Parallel execution support +│ │ +│ ├── test-cases/ # Test definitions +│ │ ├── index.ts # Test registry +│ │ ├── checks.ts # Reusable check functions +│ │ ├── utils.ts # Test utilities +│ │ ├── llm/ # LLM test cases +│ │ │ ├── basic.ts +│ │ │ ├── multi-turn.ts +│ │ │ ├── basic-error.ts +│ │ │ ├── vision.ts +│ │ │ └── long-input.ts +│ │ └── agents/ # Agent test cases +│ │ ├── basic.ts +│ │ ├── tool-call.ts +│ │ ├── tool-error.ts +│ │ ├── vision.ts +│ │ └── long-input.ts +│ │ +│ ├── runner/ # Test execution +│ │ ├── runner.ts +│ │ ├── javascript-runner.ts +│ │ ├── python-runner.ts +│ │ ├── framework-config.ts +│ │ ├── framework-discovery.ts +│ │ ├── template-renderer.ts +│ │ └── templates/ # Framework templates +│ │ +│ ├── span-collector/ # HTTP server +│ │ ├── server.ts +│ │ └── store.ts +│ │ +│ └── reporters/ # Output reporters +│ ├── ctrf-reporter.ts +│ └── live-status.ts +│ +├── dist/ # Compiled JavaScript +├── runs/ # Generated test environments (gitignored) +│ ├── js/ +│ │ └── openai-4.96.0-sentry-latest/ +│ │ ├── node_modules/ +│ │ ├── package.json +│ │ └── test-basic-llm-test.js +│ └── py/ +│ └── openai-1.82.0-sentry-latest/ +│ ├── .venv/ +│ └── test-basic-llm-test-async-streaming.py +│ +├── test-results/ # Generated reports +│ ├── ctrf-report-*.json +│ └── test-report-*.html +│ +├── docs/ # Documentation +└── archive/ # Old implementation (reference) +``` + +## Execution Flow + +1. **CLI** parses arguments, creates Orchestrator +2. **Discovery** scans `templates/` for framework `config.json` files +3. **Matrix Generation** creates test combinations: + - Framework × Test Definition × Execution Modes (sync/async, streaming/blocking) +4. **For each test run:** + - Check/create environment cache (`runs/{platform}/{framework}-{version}/`) + - Install dependencies if needed (npm install / uv sync) + - **Render** template with test definition context + Sentry DSN + - **Execute** rendered test file + - **Collect** spans from Span Collector HTTP server + - **Validate** by running each check function against spans +5. **Report** results to console + CTRF JSON + HTML + +## Template Context + +Templates receive this context when rendering: + +```javascript +{ + // From test definition + testName: "Basic LLM Test", + inputs: [{ model: "gpt-4o-mini", messages: [...] }], + agent: { name: "...", tools: [...] }, // For agent tests + causeAPIError: false, + + // From framework config + frameworkName: "openai", + + // From orchestrator + sentryDsn: "http://public@localhost:9999/123456", + + // Execution mode flags + isAsync: true, // Python only + isStreaming: false, +} +``` + +## Check Functions + +Checks are reusable validation functions defined in `src/test-cases/checks.ts`: + +```typescript +interface Check { + name: string; + fn: (spans: CapturedSpan[], config: FrameworkConfig, testDef: TestDefinition) => void; +} +``` + +### Available Checks + +**Structure:** +- `checkAISpanCount(n)` - Validate exact or range of AI span count + +**Span Attributes:** +- `checkChatSpanAttributes` - Validate chat/completion spans +- `checkAgentSpanAttributes` - Validate agent invocation spans +- `checkToolSpanAttributes` - Validate tool execution spans +- `checkAvailableTools` - Validate available_tools attribute +- `checkResponseToolCalls([...])` - Validate tool calls in response +- `checkToolCalls([...])` - Validate tool execution with input/output + +**Tokens:** +- `checkValidTokenUsage` - Token counts exist and are valid +- `checkInputTokensCached` - Cached tokens ≤ input tokens +- `checkOutputTokensReasoning` - Reasoning tokens ≤ output tokens + +**Messages:** +- `checkInputMessagesSchema` - Validate message schema +- `checkBinaryRedaction` - Binary content is redacted +- `checkMessageTrimming` - Long messages are trimmed +- `checkTrimmingMetadata` - Trimming metadata is present + +**Hierarchy:** +- `checkAgentHierarchy` - Agent span hierarchy and name propagation + +## Supported Frameworks + +### JavaScript + +| Type | Framework | Streaming | Notes | +|------|-----------|-----------|-------| +| llm | openai | both | OpenAI SDK | +| llm | anthropic | both | Anthropic SDK | +| llm | google-genai | both | Google Generative AI | +| llm | langchain | both | LangChain | +| agents | vercel | - | Vercel AI SDK | +| agents | langgraph | - | LangGraph | +| agents | mastra | - | Mastra AI Framework (uses @mastra/sentry) | + +### Python + +| Type | Framework | Streaming | Execution | +|------|-----------|-----------|-----------| +| llm | openai | both | sync/async | +| llm | anthropic | both | sync/async | +| llm | langchain | both | sync/async | +| llm | litellm | both | sync/async | +| agents | openai-agents | - | async | +| agents | langgraph | - | sync/async | +| agents | pydantic-ai | - | async | +| agents | google-genai | - | sync/async | + +## CLI Commands + +```bash +# Run all tests +npm run test run + +# List discovered frameworks +npm run test list + +# Filter by framework/platform/test +npm run test -- --framework openai +npm run test -- --platform py +npm run test -- --test "Basic LLM Test" + +# Execution mode filters +npm run test -- --streaming # Only streaming tests +npm run test -- --blocking # Only non-streaming tests +npm run test -- --sync # Only sync tests (Python) +npm run test -- --async # Only async tests (Python) + +# Parallel execution +npm run test -- -j=4 + +# Verbose output +npm run test -- --verbose + +# Use local Sentry SDK +npm run test -- --sentry-python /path/to/sentry-python +npm run test -- --sentry-javascript /path/to/sentry-javascript + +# Setup only (generate files without running) +npm run test setup -- --framework openai +``` + +## Special Framework: Mastra + +Mastra uses its own Sentry integration (`@mastra/sentry`) rather than `@sentry/node`. Key differences: + +- Uses `SentryExporter` from `@mastra/sentry` with Mastra's `Observability` system +- Attribute names follow newer OpenTelemetry conventions: + - `gen_ai.input.messages` instead of `gen_ai.request.messages` + - `gen_ai.tool.call.arguments` instead of `gen_ai.tool.input` +- Tool type is `"tool"` instead of `"function"` +- Template does not extend base.js.njk (standalone implementation) diff --git a/docs/LOCAL_SENTRY_SDK.md b/docs/LOCAL_SENTRY_SDK.md index 2fe6114..3433796 100644 --- a/docs/LOCAL_SENTRY_SDK.md +++ b/docs/LOCAL_SENTRY_SDK.md @@ -1,546 +1,86 @@ # Using Local Sentry SDK for Development -This guide explains how to use a local copy of the Sentry SDK (Python or JavaScript) when running tests, instead of installing from package repositories. This is useful when you're developing changes to the Sentry SDK and want to test them against AI SDK integrations. +This guide explains how to use local, editable installations of Sentry SDKs instead of installing from package registries. -## Quick Start +## Prerequisites -### Python SDK - -If you have the Sentry Python SDK cloned adjacent to this repository: - -```bash -# Setup all Python SDKs with local Sentry SDK -cd shared/orchestration -npm run cli setup -- --local-sentry-python ../../../sentry-python - -# Run tests with local Sentry SDK -npm run cli run -- --all --local-sentry-python ../../../sentry-python -``` - -### JavaScript SDK - -If you have the Sentry JavaScript SDK cloned adjacent to this repository: - -```bash -# Setup all JavaScript SDKs with local Sentry SDK -cd shared/orchestration -npm run cli setup -- --local-sentry-javascript ../../../sentry-javascript - -# Run tests with local Sentry SDK -npm run cli run -- --all --local-sentry-javascript ../../../sentry-javascript -``` - -### Both SDKs - -You can use both flags simultaneously to test with local versions of both SDKs: - -```bash -# Setup with both local SDKs -npm run cli setup -- --local-sentry-python ../../../sentry-python --local-sentry-javascript ../../../sentry-javascript - -# Run tests with both local SDKs -npm run cli run -- --all --local-sentry-python ../../../sentry-python --local-sentry-javascript ../../../sentry-javascript -``` - -## How It Works - -When you use the `--local-sentry-sdk` flag: - -### Python SDKs - -1. The orchestrator validates the provided path -2. For each Python SDK, it: - - Installs the local Sentry SDK as editable: `pip install -e /path/to/sentry-python` - - Installs other dependencies from `requirements.txt` (excluding sentry-sdk) -3. Tests run with your local Sentry SDK code -4. Changes to the Sentry SDK source are immediately reflected (no reinstall needed) - -**Important:** Your `requirements.txt` files remain unmodified, keeping git status clean. - -### JavaScript SDKs - -1. The orchestrator validates the provided path (must be sentry-javascript monorepo) -2. For each JavaScript SDK, it: - - Reads `package.json` to find which `@sentry/*` packages are used - - Links each package: `npm link /path/to/sentry-javascript/packages/node` - - Installs other dependencies normally -3. Tests run with your local Sentry SDK code -4. Changes to the Sentry SDK source are immediately reflected (no rebuild needed) - -**Important:** Your `package.json` files remain unmodified, keeping git status clean. - -## Setup Command - -Install all dependencies with local Sentry SDKs: - -```bash -# For Python SDK -npm run cli setup -- --local-sentry-python - -# For JavaScript SDK -npm run cli setup -- --local-sentry-javascript - -# For both -npm run cli setup -- --local-sentry-python --local-sentry-javascript -``` - -### Path Requirements - -**For Python SDK (`sentry-python`):** -- Can be relative (e.g., `../sentry-python`) or absolute -- Must point to the Sentry Python SDK repository root -- Must contain: - - `setup.py` (valid Python package) - - `sentry_sdk/` directory (the actual package) - -**For JavaScript SDK (`sentry-javascript`):** -- Can be relative (e.g., `../sentry-javascript`) or absolute -- Must point to the Sentry JavaScript SDK monorepo root -- Must contain: - - `packages/` directory (monorepo structure) - - Root `package.json` (workspace configuration) - -### Examples - -```bash -# Python SDK - Relative path (adjacent to repo) -npm run cli setup -- --local-sentry-python ../sentry-python - -# Python SDK - Absolute path -npm run cli setup -- --local-sentry-python /Users/username/dev/sentry-python - -# JavaScript SDK - Relative path (adjacent to repo) -npm run cli setup -- --local-sentry-javascript ../sentry-javascript - -# JavaScript SDK - Absolute path -npm run cli setup -- --local-sentry-javascript /Users/username/dev/sentry-javascript -``` - -## Run Command - -Run tests with local Sentry SDKs: +### Python +- **uv** package manager installed (`pip install uv` or `brew install uv`) +- Local clone of `sentry-python` repository -```bash -# For Python SDK -npm run cli run [filter] -- --local-sentry-python - -# For JavaScript SDK -npm run cli run [filter] -- --local-sentry-javascript - -# For both -npm run cli run [filter] -- --local-sentry-python --local-sentry-javascript -``` - -**Examples:** -```bash -# Run all tests with local Python SDK -npm run cli run -- --all --local-sentry-python ../sentry-python - -# Run all tests with local JavaScript SDK -npm run cli run -- --all --local-sentry-javascript ../sentry-javascript - -# Run all tests with both local SDKs -npm run cli run -- --all --local-sentry-python ../sentry-python --local-sentry-javascript ../sentry-javascript - -# Run specific Python SDK with local Sentry -npm run cli run py/openai -- --local-sentry-python ../sentry-python - -# Run specific JavaScript SDK with local Sentry -npm run cli run js/openai -- --local-sentry-javascript ../sentry-javascript - -# Run specific test case with local Python SDK -npm run cli run -- --case 1-simple --local-sentry-python ../sentry-python -``` - -## Verifying Local Installs - -### Python SDK (Editable Install) - -Check if the editable install is active: - -```bash -# From any Python SDK directory -cd sdks/py/openai -.venv/bin/pip list | grep sentry-sdk -``` - -**Expected output (editable):** -``` -sentry-sdk 2.43.0 /path/to/sentry-python -``` - -**Expected output (PyPI):** -``` -sentry-sdk 2.43.0 -``` +### JavaScript +- Local clone of `sentry-javascript` repository +- Built packages (run `yarn build` in the repo) -### JavaScript SDK (npm link) - -Check if npm link is active: - -```bash -# From any JavaScript SDK directory -cd sdks/js/openai -ls -la node_modules/@sentry/node -``` - -**Expected output (linked):** -``` -lrwxr-xr-x ... node_modules/@sentry/node -> /path/to/sentry-javascript/packages/node -``` - -**Expected output (npm registry):** -``` -drwxr-xr-x ... node_modules/@sentry/node -``` - -Or use npm to check: - -```bash -npm ls @sentry/node -# Linked shows: @sentry/node@X.Y.Z -> /path/to/sentry-javascript/packages/node -``` - -## Reverting to Package Registry Versions +## Usage ### Python SDK -To switch back to the PyPI version: +Run tests with the `--sentry-python` flag: ```bash -# Option 1: Uninstall editable, then setup normally -cd sdks/py/openai -.venv/bin/pip uninstall sentry-sdk -cd ../../../shared/orchestration -npm run cli setup - -# Option 2: Delete venv and recreate -cd sdks/py/openai -rm -rf .venv -cd ../../../shared/orchestration -npm run cli setup +npm start run -- --framework openai --sentry-python ~/sentry-python ``` ### JavaScript SDK -To switch back to the npm registry version: - -```bash -# Option 1: Unlink specific package, then setup normally -cd sdks/js/openai -npm unlink @sentry/node -cd ../../../shared/orchestration -npm run cli setup - -# Option 2: Delete node_modules and recreate -cd sdks/js/openai -rm -rf node_modules package-lock.json -cd ../../../shared/orchestration -npm run cli setup -``` - -## Upgrading Packages with Local Installs - -The upgrade command protects you from accidentally overwriting local installs. - -### Python SDK Example +Run tests with the `--sentry-javascript` flag: ```bash -npm run cli upgrade sentry-sdk 2.50.0 +npm start run -- --framework some-js-framework --sentry-javascript ~/sentry-javascript ``` -**Output when editable install is active:** -``` - py/openai - Skipping (editable install active) - To upgrade, first remove editable install: - cd sdks/py/openai && .venv/bin/pip uninstall sentry-sdk - Then run: npm run cli setup -``` - -### JavaScript SDK Example - -```bash -npm run cli upgrade @sentry/node 8.50.0 -``` - -**Output when npm link is active:** -``` - js/openai - Skipping (npm linked) - To upgrade, first unlink: - cd sdks/js/openai && npm unlink @sentry/node - Then run: npm run cli setup -``` - -### Behavior - -The upgrade command will: -- Skip all SDKs with local Sentry SDK installs (editable or linked) -- Show clear instructions on how to remove the local install -- Continue upgrading other packages normally - -## Troubleshooting - -### Python SDK Errors - -#### Error: Local Sentry SDK path does not exist - -**Problem:** The path you provided doesn't exist. - -**Solution:** Check the path and try again: -```bash -ls ../sentry-python # Should show the repository contents -``` - -#### Error: Local Sentry SDK path is not a directory - -**Problem:** The path points to a file, not a directory. - -**Solution:** Provide the repository root directory: -```bash -# Wrong ---local-sentry-sdk ../sentry-python/setup.py - -# Correct ---local-sentry-sdk ../sentry-python -``` - -#### Error: Local Sentry SDK path missing setup.py - -**Problem:** The directory isn't a valid Python package. - -**Solution:** Ensure you're pointing to the Sentry SDK repository root: -```bash -ls ../sentry-python/setup.py # Should exist -``` - -#### Error: Local Sentry SDK path missing sentry_sdk/ directory - -**Problem:** The package doesn't contain the sentry_sdk module. - -**Solution:** Make sure you're using the correct repository: -```bash -ls ../sentry-python/sentry_sdk/ # Should show the package -``` - -### JavaScript SDK Errors - -#### Error: Local Sentry JavaScript SDK path missing packages/ directory - -**Problem:** The path doesn't point to the sentry-javascript monorepo. - -**Solution:** Ensure you're pointing to the monorepo root: -```bash -ls ../sentry-javascript/packages/ # Should show all @sentry/* packages -``` - -#### Error: Local Sentry JavaScript SDK path missing root package.json - -**Problem:** The directory isn't a valid npm workspace/monorepo. - -**Solution:** Verify the monorepo structure: -```bash -ls ../sentry-javascript/package.json # Should exist -cat ../sentry-javascript/package.json | grep workspaces # Should have workspaces -``` - -#### Warning: Package not found in local SDK, using npm - -**Problem:** The JavaScript SDK doesn't have a specific package you're trying to link. - -**Solution:** This is informational - the orchestrator will fall back to npm for that package. This can happen with: -- New or experimental packages not yet in your local SDK -- Renamed packages -- Packages from other scopes - -```bash -# Verify which packages exist in your local SDK -ls ../sentry-javascript/packages/ -``` - -### General Issues - -#### Tests fail after switching to local SDK - -**Problem:** Your local Sentry SDK has breaking changes or bugs. - -**Solution for Python:** -1. Check your local Sentry SDK changes -2. Revert to PyPI version to confirm tests pass: - ```bash - cd sdks/py/openai - .venv/bin/pip uninstall sentry-sdk - .venv/bin/pip install sentry-sdk==2.43.0 - ``` - -**Solution for JavaScript:** -1. Check your local Sentry SDK changes -2. Revert to npm registry version to confirm tests pass: - ```bash - cd sdks/js/openai - npm unlink @sentry/node - npm install @sentry/node@8.0.0 - ``` - -## Common Workflows - -### Developing a Sentry SDK Feature - -**Python:** - -```bash -# 1. Setup with local SDK -npm run cli setup -- --local-sentry-python ../sentry-python - -# 2. Make changes to sentry-python code -# (edit files in ../sentry-python/) - -# 3. Run tests (changes are automatically picked up) -npm run cli run -- --all --local-sentry-python ../sentry-python - -# 4. When done, revert to PyPI version -for sdk in sdks/py/*/; do - cd "$sdk" - .venv/bin/pip uninstall sentry-sdk -y - .venv/bin/pip install sentry-sdk==2.43.0 - cd - -done -``` - -**JavaScript:** - -```bash -# 1. Setup with local SDK -npm run cli setup -- --local-sentry-javascript ../sentry-javascript - -# 2. Make changes to sentry-javascript code -# (edit files in ../sentry-javascript/packages/*) - -# 3. Run tests (changes are automatically picked up) -npm run cli run -- --all --local-sentry-javascript ../sentry-javascript - -# 4. When done, revert to npm registry version -for sdk in sdks/js/*/; do - cd "$sdk" - for pkg in node_modules/@sentry/*; do - if [ -L "$pkg" ]; then - npm unlink "$(basename $(dirname $pkg))/$(basename $pkg)" - fi - done - npm install - cd - -done -``` - -### Testing a Specific Sentry SDK Branch +## How It Works -**Python:** +When using local SDK paths: -```bash -# 1. Clone and checkout branch -cd .. -git clone https://github.com/getsentry/sentry-python.git -cd sentry-python -git checkout feature/my-new-integration +1. The CLI sets environment variables (`SENTRY_PYTHON_PATH` or `SENTRY_JAVASCRIPT_PATH`) +2. The framework's `sentryVersion` is set to `"local"` +3. Work directories use `sentry-local` instead of version number: + - Example: `runs/py/openai-1.57.0-sentry-local/` +4. Python: `uv pip install -e ` for editable install +5. JavaScript: `npm link /packages/node` to link local SDK -# 2. Setup tests with this branch -cd ../testing-ai-sdk-integrations/shared/orchestration -npm run cli setup -- --local-sentry-python ../sentry-python +## Benefits -# 3. Run tests -npm run cli run -- --all --local-sentry-python ../sentry-python -``` +- ✅ **Live changes**: Edits to local SDK are immediately reflected +- ✅ **Faster development**: No need to rebuild/reinstall after each SDK change +- ✅ **Easy debugging**: Add print statements or breakpoints in Sentry code +- ✅ **Test unreleased features**: Test local changes before they're published +- ✅ **Clear directory names**: `sentry-local` indicates local SDK usage -**JavaScript:** +## Example Output ```bash -# 1. Clone and checkout branch -cd .. -git clone https://github.com/getsentry/sentry-javascript.git -cd sentry-javascript -git checkout feature/my-new-integration +$ npm start run -- --framework openai --sentry-python ~/sentry-python -# 2. Build the SDK (important for JavaScript!) -npm install && npm run build +Using local Sentry Python SDK: /Users/you/sentry-python -# 3. Setup tests with this branch -cd ../testing-ai-sdk-integrations/shared/orchestration -npm run cli setup -- --local-sentry-javascript ../sentry-javascript +Testing 1 framework(s) with 1 test(s) -# 4. Run tests -npm run cli run -- --all --local-sentry-javascript ../sentry-javascript +[openai] Running: Basic LLM Test + Setting up Python environment in runs/py/openai-1.57.0-sentry-local... + Using uv for dependency management + ✓ pyproject.toml generated + ✓ Virtual environment created + Installing dependencies... + Installing local Sentry SDK from: /Users/you/sentry-python + ✓ Dependencies installed ``` -### Multiple Developers with Different SDK Locations +## Clearing Cache -Each developer can use their own path without affecting others: +To force reinstallation (e.g., after switching branches in sentry-python): ```bash -# Developer A (macOS) - Python SDK -npm run cli setup -- --local-sentry-python /Users/alice/dev/sentry-python - -# Developer B (Linux) - Python SDK -npm run cli setup -- --local-sentry-python /home/bob/projects/sentry-python - -# Developer C (Windows) - JavaScript SDK -npm run cli setup -- --local-sentry-javascript C:/dev/sentry-javascript - -# Developer D - Both SDKs -npm run cli setup -- --local-sentry-python ../sentry-python --local-sentry-javascript ../sentry-javascript +rm -rf runs/ +npm start run -- --framework openai --sentry-python ~/sentry-python ``` -The `requirements.txt` and `package.json` files are never modified, so there are no git conflicts. - -## Technical Details - -### Python: Why Editable Installs? - -Editable installs (`pip install -e`) create a link to your local code instead of copying files to `site-packages`. This means: -- ✅ Changes to Sentry SDK source are immediately reflected -- ✅ No need to reinstall after each change -- ✅ Easy to develop and test simultaneously +## Fallback -**Where Are Editable Installs Stored?** +If `uv` is not available for Python, the runner automatically falls back to using `pip`: -Editable install metadata is stored in: -- `.venv/lib/python3.x/site-packages/sentry-sdk.egg-link` (points to your local path) -- `.venv/lib/python3.x/site-packages/__editable__.sentry_sdk-*.pth` - -### JavaScript: Why npm link? - -npm link creates symbolic links in `node_modules` to your local code. This means: -- ✅ Changes to Sentry SDK source are immediately reflected (if already built) -- ✅ Works with TypeScript source maps for debugging -- ✅ Preserves monorepo structure - -**Where Are Links Stored?** - -npm link creates symlinks in: -- `node_modules/@sentry/node` → `/path/to/sentry-javascript/packages/node` -- `node_modules/@sentry/core` → `/path/to/sentry-javascript/packages/core` -- etc. - -**Important Note:** For TypeScript changes in sentry-javascript, you may need to rebuild: ```bash -cd ../sentry-javascript && npm run build +pip install -e "${SENTRY_PYTHON_PATH}" ``` - -### Impact on CI/Production - -**Zero impact** - local installs only affect your local development environment: -- `requirements.txt` and `package.json` files remain unchanged -- CI always installs from PyPI/npm (exact versions) -- Other developers unaffected unless they use the flag - -## Best Practices - -1. **Always specify the path** - Don't rely on default locations -2. **Use relative paths when possible** - Makes commands portable across environments -3. **Revert when done** - Switch back to PyPI versions to avoid confusion -4. **Document your setup** - Let team members know if you're using local SDK -5. **Test with PyPI version** - Before submitting PRs, verify tests pass with released SDK - -## See Also - -- [Python SDK Development Guide](https://develop.sentry.dev/sdk/python/) -- [Main Setup Documentation](../shared/orchestration/README.md) -- [Troubleshooting Guide](./TROUBLESHOOTING.md) diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md deleted file mode 100644 index 83a8dd1..0000000 --- a/docs/TROUBLESHOOTING.md +++ /dev/null @@ -1,505 +0,0 @@ -# Troubleshooting Guide - -This guide covers common issues you'll encounter when working with the Sentry AI SDK testing framework and how to resolve them. - -## Quick Diagnostic Checklist - -When a test fails, check these first: - -- [ ] Did you run `npm install` in the SDK directory? -- [ ] Did you create a `.venv` and install requirements (Python SDKs)? -- [ ] Are relative import paths correct? (count `../` carefully) -- [ ] Is `sys.path.insert(0, ...)` present in Python setup.py? -- [ ] Are you using `.js` files (not `.ts`) for SDK tests? -- [ ] Is `framework_type` in `config.json` set correctly for your SDK? -- [ ] Did you call `await Sentry.flush()` before assertions (JavaScript)? - -## Common Pitfalls - -### Pitfall #1: Missing or incorrect config.json - -**Symptoms:** - -- Error: "Framework type not specified" -- Error: "Cannot find config.json" -- Test uses wrong fixture variant - -**Problem:** - -- Missing `config.json` in SDK directory -- `framework_type` field not set -- Wrong framework type specified - -**Solution:** - -```json -// ✅ GOOD - create config.json in SDK directory -// In: sdks/js/vercel/config.json -{ - "sdk_name": "vercel", - "framework_type": "agentic", - "overrides": {} -} -``` - -**For low-level SDKs:** - -```json -{ - "sdk_name": "openai", - "framework_type": "low-level", - "overrides": { - "1-simple": { - "model": "gpt-5-nano", - "gen_ai.request.model": "gpt-5-nano", - "gen_ai.response.model": "gpt-5-nano" - } - } -} -``` - -**Additional notes:** - -- Every SDK must have a `config.json` file -- Framework type is determined by the SDK's architecture (see CLAUDE.md) -- All test cases in an SDK use the same framework type from config -- Use `overrides` to customize model names per test case - ---- - -### Pitfall #2: Using ES modules (import/export) in SDK test files - -**Symptoms:** - -- SyntaxError: Cannot use import statement outside a module -- ReferenceError: exports is not defined - -**Problem:** - -```javascript -// ❌ BAD - causes "Cannot use import statement outside a module" -import Sentry from "@sentry/node"; -import { loadFixture } from "../../_test-utils/fixtures"; - -export default async function () { - // ... -} -``` - -**Why it breaks:** SDK test files use CommonJS for compatibility with the test runner and to avoid compilation steps. - -**Solution:** - -```javascript -// ✅ GOOD - use CommonJS require/module.exports -const Sentry = require("@sentry/node"); -const { loadFixture } = require("../../_test-utils/fixtures"); - -module.exports = async function () { - // test implementation -}; -``` - -**Module system reference:** -| Location | Module System | Syntax | -|----------|---------------|--------| -| `sdks/js/*/` | CommonJS | `require()`, `module.exports` | -| `shared/orchestration/` | ES Modules | `import`, `export` | -| `sdks/js/_test-utils/` | CommonJS | `require()`, `module.exports` | - ---- - -### Pitfall #3: Forgetting sys.path setup in Python SDK - -**Symptoms:** - -- ModuleNotFoundError: No module named 'mock_transport' -- ModuleNotFoundError: No module named 'test_runner' -- ModuleNotFoundError: No module named 'fixture_loader' - -**Problem:** - -```python -# ❌ BAD - import fails immediately -import sentry_sdk -from mock_transport import create_mock_transport # ERROR -``` - -**Why it breaks:** Python doesn't have project-wide module resolution by default. Shared test utilities are outside the SDK directory, so Python can't find them. - -**Solution:** - -```python -# ✅ GOOD - add sys.path setup BEFORE imports -import os -import sys -import sentry_sdk -from pathlib import Path -from dotenv import load_dotenv - -# Add shared test utils to path (CRITICAL - DO NOT FORGET) -shared_path = Path(__file__).parent.parent.parent.parent / "_test-utils" -sys.path.insert(0, str(shared_path)) - -# Now imports work -from mock_transport import create_mock_transport, get_mock_transport, clear_mock_transport -``` - -**Path breakdown:** - -``` -Path(__file__) # /path/to/sdks/py/your-sdk/setup.py -.parent # /path/to/sdks/py/your-sdk/ -.parent.parent.parent.parent # /path/to/ (repo root) -/ "_test-utils" # /path/to/shared/test-utils/py/ -``` - ---- - -### Pitfall #4: Wrong relative path counts in JavaScript - -**Symptoms:** - -- Error: Cannot find module '../../\_test-utils/fixtures' -- Module not found: Can't resolve '../\_test-utils/mock-transport' - -**Problem:** - -```javascript -// ❌ BAD - wrong number of ../ -const { loadFixture } = require("../_test-utils/fixtures"); -// Too few levels up! -``` - -**Why it breaks:** Relative paths must traverse up to repo root, then down to target directory. - -**Solution:** - -```javascript -// ✅ GOOD - count levels carefully -// From: sdks/js/vercel/cases/1-simple.js (4 levels deep from root) -// To: sdks/js/_test-utils/fixtures/ -const { loadFixture } = require("../../_test-utils/fixtures"); -// ^^^^ -// 4 levels: cases/ -> vercel/ -> js/ -> sdks/ -> root -``` - -**Formula for counting:** - -1. Start at your test file location -2. Count how many directories deep you are from repo root -3. That's your `../` count -4. Then add the path down to target directory - -**Common paths from test cases:** - -```javascript -// From sdks/js/{sdk}/cases/{test}.js: -require("../../_test-utils/mock-transport.js"); -require("../../_test-utils/fixtures"); -require("../setup"); // SDK's setup.js - -// From sdks/js/{sdk}/setup.js: -require("../_test-utils/mock-transport.js"); -``` - ---- - -### Pitfall #5: Missing .venv in Python SDK - -**Symptoms:** - -- Error: No module named 'sentry_sdk' -- Error: No module named 'openai' (or other AI SDK) -- Python test exits immediately with import errors - -**Problem:** - -```bash -$ npm run cli run -- --sdk py/your-sdk -# Error: No module named 'sentry_sdk' -``` - -**Why it breaks:** Python dependencies aren't installed in the system Python or aren't accessible to the test runner. - -**Solution:** - -```bash -# ✅ GOOD - create virtual environment first -cd sdks/py/your-sdk -python3 -m venv .venv -source .venv/bin/activate # On Windows: .venv\Scripts\activate -pip install -r requirements.txt - -# Now run tests from orchestration directory -cd ../../../shared/orchestration -npm run cli run -- --sdk py/your-sdk -``` - -**How the orchestrator finds Python:** - -1. Checks for `sdks/py/your-sdk/.venv/bin/python` ← Uses this if exists -2. Falls back to `python3` system command - -**Best practice:** Always create a `.venv` in each Python SDK directory to ensure isolated dependencies. - ---- - -### Pitfall #6: Using TypeScript (.ts) for SDK test files - -**Symptoms:** - -- Warning: "SDK has no setup file" (✓ indicator missing in `list` output) -- Tests don't run even though files exist - -**Problem:** - -```bash -# Created setup.ts instead of setup.js -sdks/js/your-sdk/ -├── setup.ts # ❌ Wrong! -└── cases/ - └── 1-simple.ts # ❌ Wrong! -``` - -**Why it breaks:** The orchestrator only looks for `.js` files (and `.py` for Python SDKs). TypeScript files require compilation. - -**Solution:** - -```bash -# ✅ GOOD - use .js for SDK files -sdks/js/your-sdk/ -├── setup.js # ✓ Correct -└── cases/ - └── 1-simple.js # ✓ Correct -``` - -**File extension reference:** -| Location | Extension | Reason | -|----------|-----------|--------| -| `sdks/js/*/` | `.js` | No build step, CommonJS compatibility | -| `shared/orchestration/src/` | `.ts` | Type safety for complex orchestration logic | -| `sdks/py/*/` | `.py` | Standard Python | - ---- - -### Pitfall #7: Calling assert_sentry() from main() in Python - -**Symptoms:** - -- AssertionError: No spans found (but SDK is instrumented correctly) -- Validation fails even though test logic succeeds - -**Problem:** - -```python -# ❌ BAD - assert_sentry() runs too early -async def main(): - await run_test() - await assert_sentry() # Mock transport not fully populated yet! -``` - -**Why it breaks:** The orchestrator needs to call `sentry_sdk.flush()` between running the test and checking assertions. If you call `assert_sentry()` from `main()`, it runs before flushing completes. - -**Solution:** - -```python -# ✅ GOOD - let orchestrator call them separately -async def main(): - """Entry point - runs ONLY the test logic""" - print(" Running 1-simple: Basic Completion") - await run_test() - print(" ✓ Test logic completed") - -async def assert_sentry(): - """Validation - checks ONLY Sentry captured data - Called by orchestrator AFTER main() completes and Sentry flushes""" - await asyncio.sleep(0.1) # Buffer for transport - await assert_sentry_captured() - print(" ✓ 1-simple validation passed") -``` - -**Test execution flow for Python:** - -1. Orchestrator imports test module -2. Orchestrator calls `main()` ← Run test logic -3. Orchestrator calls `sentry_sdk.flush()` ← Ensure events captured -4. Orchestrator calls `assert_sentry()` ← Validate captured data - ---- - -### Pitfall #8: Wrong framework type for SDK - -**Symptoms:** - -- Error: "No span found with op='gen_ai.invoke_agent'" (when using agentic fixture on low-level SDK) -- Error: "No span found with op='gen_ai.chat'" (when using low-level fixture on agentic SDK) - -**Problem:** -Test fails even though the SDK is working correctly. The issue is using the wrong fixture variant. - -**Solution:** - -1. Run your SDK and examine actual spans captured -2. Check if you see agent/workflow wrapper spans → use `FRAMEWORK_TYPE = "agentic"` -3. Check if you only see direct LLM call spans → use `FRAMEWORK_TYPE = "low-level"` -4. Update `FRAMEWORK_TYPE` constant in **all** test cases for that SDK - -**Framework type examples:** - -```javascript -// Agentic frameworks (produce wrapper spans) -const FRAMEWORK_TYPE = "agentic"; -// Examples: Vercel AI SDK, OpenAI Agents SDK - -// Low-level frameworks (direct LLM calls only) -const FRAMEWORK_TYPE = "low-level"; -// Examples: OpenAI SDK, Anthropic SDK -``` - -**How to diagnose:** - -```javascript -// Add debug output in your test -const transport = getMockSentryTransport(); -const spans = transport.getSpans(); -console.log( - "Captured spans:", - spans.map((s) => s.op) -); - -// Output might be: -// ["gen_ai.invoke_agent", "gen_ai.chat"] → agentic -// ["gen_ai.chat"] → low-level -``` - ---- - -## Debugging Tips - -### Enable Verbose Logging - -**JavaScript:** - -```javascript -// In test case -console.log("Debug: Running test with model:", model); -console.log( - "Debug: Captured spans:", - transport.getSpans().map((s) => ({ op: s.op, name: s.description })) -); -``` - -**Python:** - -```python -# In test case -print(f"Debug: Running test with model: {model}") -transport = get_mock_sentry_transport() -print(f"Debug: Captured {len(transport.get_spans())} spans") -``` - -### Inspect Captured Sentry Data - -```javascript -// JavaScript -const transport = getMockSentryTransport(); -console.log("Spans:", JSON.stringify(transport.getSpans(), null, 2)); -console.log( - "Transactions:", - JSON.stringify(transport.getTransactions(), null, 2) -); -console.log("Events:", JSON.stringify(transport.getEvents(), null, 2)); -``` - -```python -# Python -transport = get_mock_sentry_transport() -import json -print("Spans:", json.dumps(transport.get_spans(), indent=2, default=str)) -``` - -### Verify Sentry Is Initialized - -```javascript -// JavaScript -const Sentry = require("@sentry/node"); -const client = Sentry.getClient(); -console.log("Sentry client:", client ? "initialized" : "NOT initialized"); -``` - -```python -# Python -import sentry_sdk -client = sentry_sdk.Hub.current.client -print(f"Sentry client: {'initialized' if client else 'NOT initialized'}") -``` - ---- - -## Error Message Decoder - -### "Mock transport not initialized" - -**Likely causes:** - -1. Forgot to call `createMockTransport()` before `Sentry.init()` (JavaScript) -2. Exported frameworkType from setup.js (see Pitfall #1) -3. Module loading order issue - -**Fix:** Ensure mock transport is created before Sentry.init() - ---- - -### "Cannot find module" - -**Likely causes:** - -1. Wrong relative path (see Pitfall #4) -2. Missing `sys.path.insert()` in Python (see Pitfall #3) -3. Forgot to run `npm install` or `pip install` - -**Fix:** Count `../` carefully and verify imports - ---- - -### "No span found with op='...'" - -**Likely causes:** - -1. Wrong framework type (see Pitfall #8) -2. SDK integration not working (check Sentry is initialized) -3. Forgot to flush Sentry before assertions - -**Fix:** Verify spans are captured, check framework type - ---- - -### "Fixture validation failed" - -**Likely causes:** - -1. Fixture expectations don't match actual SDK output -2. Sentry SDK version mismatch -3. AI SDK version changed behavior - -**Fix:** Compare actual attributes vs required attributes in error message - ---- - -## Getting Help - -If you're still stuck after checking this guide: - -1. **Check the error message carefully** - It usually tells you exactly what's wrong -2. **Compare with working examples** - Look at `sdks/js/vercel/` or `sdks/py/openai-agents/` -3. **Verify basic setup** - Can you run existing tests successfully? -4. **Isolate the issue** - Does it happen with all tests or just one? - -## See Also - -- [Adding SDKs Guide](../sdks/README.md) - Step-by-step SDK implementation -- [Test Utilities (JS)](../sdks/js/_test-utils/README.md) - Mock transport and fixtures -- [Test Utilities (Python)](../sdks/py/_test-utils/README.md) - Mock transport and fixtures -- [CLI Documentation](../shared/orchestration/README.md) - Running tests -- [Main Documentation](../CLAUDE.md) - Project overview diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..81a4977 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,1150 @@ +{ + "name": "sentry-ai-sdks-test", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "sentry-ai-sdks-test", + "version": "1.0.0", + "dependencies": { + "@hono/node-server": "^1.19.9", + "@types/nunjucks": "^3.2.6", + "chai": "^5.1.2", + "cli-spinners": "^3.4.0", + "ctrf": "^0.0.17", + "dotenv": "^17.2.3", + "hono": "^4.11.5", + "htm": "^3.1.1", + "log-update": "^7.0.2", + "nunjucks": "^3.2.4", + "prettier": "^3.8.1", + "vhtml": "^2.2.0" + }, + "devDependencies": { + "@types/chai": "^5.2.3", + "@types/node": "^22.10.5", + "@types/vhtml": "^2.2.9", + "typescript": "^5.7.3" + } + }, + "node_modules/@hono/node-server": { + "version": "1.19.9", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.9.tgz", + "integrity": "sha512-vHL6w3ecZsky+8P5MD+eFfaGTyCeOHUIFYMGpQGbrBTSmNNoxv0if69rEZ5giu36weC5saFuznL411gRX7bJDw==", + "license": "MIT", + "engines": { + "node": ">=18.14.1" + }, + "peerDependencies": { + "hono": "^4" + } + }, + "node_modules/@isaacs/balanced-match": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", + "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==", + "license": "MIT", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@isaacs/brace-expansion": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz", + "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==", + "license": "MIT", + "dependencies": { + "@isaacs/balanced-match": "^4.0.1" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.19.7", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.7.tgz", + "integrity": "sha512-MciR4AKGHWl7xwxkBa6xUGxQJ4VBOmPTF7sL+iGzuahOFaO0jHCsuEfS80pan1ef4gWId1oWOweIhrDEYLuaOw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/nunjucks": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@types/nunjucks/-/nunjucks-3.2.6.tgz", + "integrity": "sha512-pHiGtf83na1nCzliuAdq8GowYiXvH5l931xZ0YEHaLMNFgynpEqx+IPStlu7UaDkehfvl01e4x/9Tpwhy7Ue3w==", + "license": "MIT" + }, + "node_modules/@types/vhtml": { + "version": "2.2.9", + "resolved": "https://registry.npmjs.org/@types/vhtml/-/vhtml-2.2.9.tgz", + "integrity": "sha512-maEIRJb+PdK2FWDORl0a0aNUSGlniMl8pN+7WbanLzx1Gry4gLfsT0C9O/6JucPPBHCNrqDSImr2QcsUENLKUg==", + "dev": true, + "license": "MIT" + }, + "node_modules/a-sync-waterfall": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/a-sync-waterfall/-/a-sync-waterfall-1.0.1.tgz", + "integrity": "sha512-RYTOHHdWipFUliRFMCS4X2Yn2X8M87V/OpSqWzKKOGhzqyUxzyVmhHDH9sAvG+ZuQf/TAOFsLCpMw09I1ufUnA==", + "license": "MIT" + }, + "node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ansi-escapes": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.2.0.tgz", + "integrity": "sha512-g6LhBsl+GBPRWGWsBtutpzBYuIIdBkLEvad5C/va/74Db018+5TZiyA26cZJAr3Rft5lprVqOIPxf5Vid6tqAw==", + "license": "MIT", + "dependencies": { + "environment": "^1.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", + "license": "MIT" + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/chai": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", + "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "license": "MIT", + "dependencies": { + "assertion-error": "^2.0.1", + "check-error": "^2.1.1", + "deep-eql": "^5.0.1", + "loupe": "^3.1.0", + "pathval": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/check-error": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", + "integrity": "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==", + "license": "MIT", + "engines": { + "node": ">= 16" + } + }, + "node_modules/cli-cursor": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-5.0.0.tgz", + "integrity": "sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==", + "license": "MIT", + "dependencies": { + "restore-cursor": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-spinners": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-3.4.0.tgz", + "integrity": "sha512-bXfOC4QcT1tKXGorxL3wbJm6XJPDqEnij2gQ2m7ESQuE+/z9YFIWnl/5RpTiKWbMq3EVKR4fRLJGn6DVfu0mpw==", + "license": "MIT", + "engines": { + "node": ">=18.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cliui": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", + "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==", + "license": "ISC", + "dependencies": { + "string-width": "^7.2.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "license": "MIT" + }, + "node_modules/cliui/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/commander": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", + "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==", + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/ctrf": { + "version": "0.0.17", + "resolved": "https://registry.npmjs.org/ctrf/-/ctrf-0.0.17.tgz", + "integrity": "sha512-PPk9b+AuA+UoBcbzSQWXMIuh5601zDHgXlmHIG8ESxTUnnb0eM2sz8H3jQLYQZTpyIaZVCLFZFslIDB1EMVZ1g==", + "license": "MIT", + "dependencies": { + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "glob": "^11.0.3", + "typescript": "^5.8.3", + "yargs": "^18.0.0" + }, + "bin": { + "ctrf": "dist/cli.js" + }, + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/deep-eql": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", + "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/dotenv": { + "version": "17.2.3", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", + "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "license": "MIT" + }, + "node_modules/environment": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", + "integrity": "sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", + "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.4.0.tgz", + "integrity": "sha512-QZjmEOC+IT1uk6Rx0sX22V6uHWVwbdbxf1faPqJ1QhLdGgsRGCZoyaQBm/piRdJy/D2um6hM1UP7ZEeQ4EkP+Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-11.1.0.tgz", + "integrity": "sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "foreground-child": "^3.3.1", + "jackspeak": "^4.1.1", + "minimatch": "^10.1.1", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^2.0.0" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/hono": { + "version": "4.11.5", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.11.5.tgz", + "integrity": "sha512-WemPi9/WfyMwZs+ZUXdiwcCh9Y+m7L+8vki9MzDw3jJ+W9Lc+12HGsd368Qc1vZi1xwW8BWMMsnK5efYKPdt4g==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=16.9.0" + } + }, + "node_modules/htm": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/htm/-/htm-3.1.1.tgz", + "integrity": "sha512-983Vyg8NwUE7JkZ6NmOqpCZ+sh1bKv2iYTlUkzlWmA5JD2acKoxd4KVxbMmxX/85mtfdnDmTFoNKcg5DGAvxNQ==", + "license": "Apache-2.0" + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jackspeak": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.1.tgz", + "integrity": "sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/log-update": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/log-update/-/log-update-7.0.2.tgz", + "integrity": "sha512-cSSF1K5w9juI2+JeSRAdaTUZJf6cJB0aWwWO1nQQkcWw44+bIfXmhZMwK2eEsv6tXvU3UfKX/kzcX6SP+1tLAw==", + "license": "MIT", + "dependencies": { + "ansi-escapes": "^7.1.0", + "cli-cursor": "^5.0.0", + "slice-ansi": "^7.1.2", + "strip-ansi": "^7.1.2", + "wrap-ansi": "^9.0.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-update/node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "license": "MIT" + }, + "node_modules/log-update/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-update/node_modules/wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/loupe": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", + "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "11.2.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.5.tgz", + "integrity": "sha512-vFrFJkWtJvJnD5hg+hJvVE8Lh/TcMzKnTgCWmtBipwI5yLX/iX+5UB2tfuyODF5E7k9xEzMdYgGqaSb1c0c5Yw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/mimic-function": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/mimic-function/-/mimic-function-5.0.1.tgz", + "integrity": "sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", + "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/brace-expansion": "^5.0.0" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/nunjucks": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/nunjucks/-/nunjucks-3.2.4.tgz", + "integrity": "sha512-26XRV6BhkgK0VOxfbU5cQI+ICFUtMLixv1noZn1tGU38kQH5A5nmmbk/O45xdyBhD1esk47nKrY0mvQpZIhRjQ==", + "license": "BSD-2-Clause", + "dependencies": { + "a-sync-waterfall": "^1.0.0", + "asap": "^2.0.3", + "commander": "^5.1.0" + }, + "bin": { + "nunjucks-precompile": "bin/precompile" + }, + "engines": { + "node": ">= 6.9.0" + }, + "peerDependencies": { + "chokidar": "^3.3.0" + }, + "peerDependenciesMeta": { + "chokidar": { + "optional": true + } + } + }, + "node_modules/onetime": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-7.0.0.tgz", + "integrity": "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==", + "license": "MIT", + "dependencies": { + "mimic-function": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.1.tgz", + "integrity": "sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/pathval": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", + "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", + "license": "MIT", + "engines": { + "node": ">= 14.16" + } + }, + "node_modules/prettier": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.1.tgz", + "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/restore-cursor": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-5.1.0.tgz", + "integrity": "sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==", + "license": "MIT", + "dependencies": { + "onetime": "^7.0.0", + "signal-exit": "^4.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/slice-ansi": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-7.1.2.tgz", + "integrity": "sha512-iOBWFgUX7caIZiuutICxVgX1SdxwAVFFKwt1EvMYYec/NWO5meOJ6K5uQxhrYBdQJne4KxiqZc+KptFOWFSI9w==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "is-fullwidth-code-point": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/slice-ansi/node_modules/is-fullwidth-code-point": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-5.1.0.tgz", + "integrity": "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.3.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", + "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/vhtml": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/vhtml/-/vhtml-2.2.0.tgz", + "integrity": "sha512-TPXrXrxBOslRUVnlVkiAqhoXneiertIg86bdvzionrUYhEuiROvyPZNiiP6GIIJ2Q7oPNVyEtIx8gMAZZE9lCQ==", + "license": "MIT" + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.0.0.tgz", + "integrity": "sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg==", + "license": "MIT", + "dependencies": { + "cliui": "^9.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "string-width": "^7.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^22.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "node_modules/yargs-parser": { + "version": "22.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", + "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", + "license": "ISC", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "license": "MIT" + }, + "node_modules/yargs/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/package.json b/package.json index d2b0be4..b0346cb 100644 --- a/package.json +++ b/package.json @@ -5,13 +5,38 @@ "private": true, "type": "module", "scripts": { - "cli": "cd shared/orchestration && npm run cli --" + "build": "tsc", + "postbuild": "rm -rf dist/runner/templates && mkdir -p dist/runner/templates && cp -r src/runner/templates/* dist/runner/templates/", + "test": "node dist/cli.js", + "setup": "node dist/setup.js", + "dev": "npm run build && node dist/cli.js", + "report": "node dist/generate-html-report.js" }, "keywords": [ "sentry", "ai", "testing" ], + "dependencies": { + "@hono/node-server": "^1.19.9", + "@types/nunjucks": "^3.2.6", + "chai": "^5.1.2", + "cli-spinners": "^3.4.0", + "ctrf": "^0.0.17", + "dotenv": "^17.2.3", + "hono": "^4.11.5", + "htm": "^3.1.1", + "log-update": "^7.0.2", + "nunjucks": "^3.2.4", + "prettier": "^3.8.1", + "vhtml": "^2.2.0" + }, + "devDependencies": { + "@types/chai": "^5.2.3", + "@types/node": "^22.10.5", + "@types/vhtml": "^2.2.9", + "typescript": "^5.7.3" + }, "volta": { "node": "22.22.0" } diff --git a/sdks/README.md b/sdks/README.md deleted file mode 100644 index a18789b..0000000 --- a/sdks/README.md +++ /dev/null @@ -1,510 +0,0 @@ -# Adding & Implementing SDKs - -This guide covers how to add new SDK implementations to the testing framework. - -## Quick Reference Checklist - -When adding a new SDK, ensure you: - -- [ ] Create `config.json` with framework type and model overrides -- [ ] Pin **exact versions** in `package.json` or `requirements.txt` (no `^`, `~`, or `>=`) -- [ ] Use `runTestCase` helper from `test-runner.cjs` (JS) or `test_runner.py` (Python) -- [ ] **Never hardcode model mappings** - use config.json overrides -- [ ] Follow existing SDK patterns (`vercel`, `anthropic` for JS; `google-genai` for Python) -- [ ] Test with `npm run cli run {language}/{sdk-name}` - -## Currently Implemented - -| Language | SDK | Framework Type | Test Cases | -| ---------- | --------------- | -------------- | ---------- | -| JavaScript | `vercel` | agentic | 1-simple | -| JavaScript | `openai` | low-level | 1-simple | -| JavaScript | `anthropic` | low-level | 1-simple | -| JavaScript | `langchain` | low-level | 1-simple | -| JavaScript | `langgraph` | agentic | 1-simple | -| JavaScript | `google-genai` | low-level | 1-simple | -| Python | `openai` | low-level | 1-simple | -| Python | `openai-agents` | agentic | 1-simple | -| Python | `anthropic` | low-level | 1-simple | -| Python | `langchain` | low-level | 1-simple | -| Python | `langgraph` | agentic | 1-simple | -| Python | `google-genai` | low-level | 1-simple | -| Python | `litellm` | low-level | 1-simple | -| Python | `pydantic-ai` | agentic | 1-simple | - -## Adding a New JavaScript SDK - -### Step 1: Determine Framework Type - -First, determine if your SDK is "agentic" or "low-level": - -- Run a simple test and examine spans -- Agent/workflow wrappers → `agentic` -- Direct LLM calls only → `low-level` - -See [../shared/specs/README.md](../shared/specs/README.md) for framework type definitions. - -### Step 2: Create SDK Configuration - -**IMPORTANT:** Every SDK must have a `config.json` file to define its framework type and any fixture overrides. - -#### Why Config Files Are Required - -The test framework uses JSON fixtures to define expected behavior. However, different SDKs have different requirements: - -- **Model names**: OpenAI SDKs use `gpt-5-nano`, Google GenAI uses `gemini-2.5-flash-lite`, Anthropic uses `claude-3-5-sonnet-20241022` -- **Span attributes**: Some SDKs may capture different attributes or use different attribute names -- **Per-test-case variation**: Some tests may need different models (e.g., use a more capable model for complex agentic workflows) - -The config.json file allows per-SDK parametrization of test fixtures without duplicating fixture files. - -#### Config File Format - -Create `sdks/{language}/{sdk-name}/config.json`: - -```json -{ - "sdk_name": "your-sdk", - "framework_type": "low-level", - "overrides": { - "1-simple": { - "model": "your-model-name", - "gen_ai.request.model": "your-model-name", - "gen_ai.response.model": "your-model-name" - } - } -} -``` - -**Schema:** - -- `sdk_name` (required): Unique identifier for your SDK -- `framework_type` (required): Either `"agentic"` or `"low-level"` -- `overrides` (optional): Per-test-case overrides for fixture values - -**Overrides Format:** - -- Key = test case ID (e.g., `"1-simple"`, `"2-simple-with-error"`) -- Value = object with attribute overrides -- Special key `"model"` automatically overrides `inputs.model` -- Other keys override `expectations.spans[].required_attributes` with matching keys - -#### Examples - -**OpenAI SDK (no overrides needed):** - -```json -{ - "sdk_name": "openai", - "framework_type": "low-level", - "overrides": {} -} -``` - -**Google GenAI SDK (different model):** - -```json -{ - "sdk_name": "google-genai", - "framework_type": "low-level", - "overrides": { - "1-simple": { - "model": "gemini-2.5-flash-lite", - "gen_ai.request.model": "gemini-2.5-flash-lite", - "gen_ai.response.model": "gemini-2.5-flash-lite" - } - } -} -``` - -**Anthropic SDK (different model):** - -```json -{ - "sdk_name": "anthropic", - "framework_type": "low-level", - "overrides": { - "1-simple": { - "model": "claude-3-5-sonnet-20241022", - "gen_ai.request.model": "claude-3-5-sonnet-20241022", - "gen_ai.response.model": "claude-3-5-sonnet-20241022" - } - } -} -``` - -**Per-Test-Case Variation:** - -```json -{ - "sdk_name": "openai", - "framework_type": "low-level", - "overrides": { - "1-simple": { - "model": "gpt-5-nano" - }, - "2-multi-step": { - "model": "gpt-5-nano" - } - } -} -``` - -See [../shared/specs/sdk-config-schema.json](../shared/specs/sdk-config-schema.json) for the complete JSON schema. - -### Step 3: Create Directory Structure - -```bash -mkdir -p sdks/js/{sdk-name}/cases -touch sdks/js/{sdk-name}/setup.js -touch sdks/js/{sdk-name}/package.json -``` - -### Step 3: Create package.json - -**IMPORTANT: Always use exact latest versions (no ^ or ~)** - -To get the latest versions, run: - -```bash -npm view @sentry/node version -npm view {your-ai-sdk} version -npm view dotenv version -``` - -Create `package.json` with **exact versions** (no semver ranges): - -```json -{ - "name": "@sentry-ai-sdks/{sdk-name}", - "version": "1.0.0", - "description": "{SDK Name} integration tests for Sentry", - "dependencies": { - "@sentry/node": "10.24.0", - "{your-ai-sdk}": "x.x.x", - "dotenv": "16.4.7" - } -} -``` - -**Why exact versions?** - -- Ensures reproducible builds -- Makes it clear when dependencies need updating -- Prevents unexpected breaking changes -- Easier to track which versions are being tested - -Run `npm install` in the SDK directory. - -### Step 4: Implement setup.js (Copy-Paste Template) - -**CRITICAL: Follow this exact pattern - check existing SDKs like `vercel` or `anthropic` for current examples** - -```javascript -/** - * Setup file for {SDK Name} tests - * - * Initializes Sentry with {SDK Name}-specific integrations. - */ - -const Sentry = require("@sentry/node"); -const { config } = require("dotenv"); -const { resolve } = require("path"); -const { createMockTransport } = require("../_test-utils/mock-transport.cjs"); - -// Load environment variables -config({ path: resolve(__dirname, ".env") }); - -// Initialize Sentry -// Note: This template uses @sentry/node where AI integrations are auto-enabled. -// If using a different Sentry package (e.g., @sentry/browser), You must use -// manual instrumentation techniques to capture AI agent spans. -Sentry.init({ - dsn: process.env.SENTRY_DSN || "https://public@127.0.0.1/1", - tracesSampleRate: 1.0, - transport: createMockTransport, - sendDefaultPii: true, -}); - -module.exports = { Sentry }; -``` - -**How to find the correct integration name:** - -If you need to manually add integrations (e.g., for a different Sentry package), you can discover available AI integrations: - -```bash -node -e "const Sentry = require('@sentry/node'); console.log(Object.keys(Sentry).filter(k => k.toLowerCase().includes('ai')).join('\n'))" -``` - -### Step 5: Implement Test Case (e.g., 1-simple.js) - -**CRITICAL: Use the `runTestCase` helper from test-runner.cjs** - -The framework type is loaded automatically from `config.json`, so you don't need to specify it in test cases. - -```javascript -/** - * 1-simple: Basic Completion - * - * Tests a simple chat completion request with {Your SDK} - * and verifies that Sentry captures the appropriate spans and AI monitoring data. - */ - -const { Sentry } = require("../setup"); -const YourSDK = require("your-sdk-package"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, system, prompt } = inputs; - - // Your SDK-specific code here - // Example for Anthropic: - const client = new YourSDK({ - apiKey: process.env.YOUR_API_KEY, - }); - - const response = await client.yourMethod({ - model: model, // model is already overridden via config.json - system: system, - messages: [{ role: "user", content: prompt }], - }); - - // Validate response - if (!response) { - throw new Error("No completion returned"); - } - - // Optional: Log for debugging - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${JSON.stringify(response)}`); - } -} - -// Framework type is loaded from config.json automatically -module.exports = runTestCase("1-simple", testLogic, Sentry); -``` - -**IMPORTANT: Key Points** - -1. **Never hardcode model mappings** in test files - always use config.json overrides -2. **Pin exact versions** in package.json (no `^` or `~`) -3. **Use the runTestCase helper** - don't manually handle fixtures/validation -4. **Check existing SDKs** (`vercel`, `anthropic`) for current patterns before implementing - -### Step 6: Test Your Implementation - -```bash -# Run your SDK's tests -cd shared/orchestration -npm run cli run -- --sdk js/{your-sdk} - -# Run all tests to see your SDK in the matrix -npm run cli run -- --all -``` - -## Adding a New Python SDK - -### Step 1: Determine Framework Type - -Same as JavaScript - determine if "agentic" or "low-level". - -### Step 2: Create SDK Configuration - -**Same as JavaScript Step 2** - Create `config.json` with framework type and overrides. - -See examples in the JavaScript section above. - -### Step 3: Create Directory Structure - -```bash -mkdir -p sdks/py/{sdk-name}/cases -touch sdks/py/{sdk-name}/setup.py -touch sdks/py/{sdk-name}/requirements.txt -``` - -### Step 3: Create requirements.txt - -**IMPORTANT: Always use exact latest versions (use == not >=)** - -To get the latest versions, run: - -```bash -pip index versions sentry-sdk -pip index versions {your-ai-sdk} -pip index versions python-dotenv -``` - -Create `requirements.txt` with **exact versions**: - -``` -sentry-sdk==2.x.x -{your-ai-sdk}==x.x.x -python-dotenv==1.x.x -``` - -**Why exact versions?** - -- Ensures reproducible builds -- Makes it clear when dependencies need updating -- Prevents unexpected breaking changes -- Easier to track which versions are being tested - -Create virtual environment and install: - -```bash -cd sdks/py/{sdk-name} -python3 -m venv .venv -source .venv/bin/activate -pip install -r requirements.txt -``` - -### Step 4: Implement setup.py (Copy-Paste Template) - -```python -""" -Setup file for {SDK Name} tests -""" - -import os -import sys -import sentry_sdk -from pathlib import Path -from dotenv import load_dotenv - -# Add test utils to path (CRITICAL - DO NOT FORGET) -test_utils_path = Path(__file__).parent.parent / "_test-utils" -sys.path.insert(0, str(test_utils_path)) - -from mock_transport import create_mock_transport, get_mock_transport, clear_mock_transport - - -def before_all(): - """Initialize Sentry with mock transport""" - print("🔧 Setting up {SDK Name} tests...") - - # Load environment variables - env_path = Path(__file__).parent / ".env" - load_dotenv(dotenv_path=env_path) - - # Pre-initialize mock transport - from mock_transport import MockTransportCapture, _mock_transport_capture - import mock_transport as mt - mt._mock_transport_capture = MockTransportCapture() - - mock_transport_instance = create_mock_transport( - options={"dsn": os.getenv("SENTRY_DSN", "https://public@127.0.0.1/1")} - ) - - # Initialize Sentry - # Note: AI integrations are auto-enabled in Python - no need to manually add them - sentry_sdk.init( - traces_sample_rate=1.0, - transport=mock_transport_instance, - ) - - print(" ✓ Sentry initialized with mock transport") - - -def before_each(): - """Reset test state""" - print(" ↻ Resetting test state...") - clear_mock_transport() - - -def after_each(): - """Clean up after test""" - print(" ✓ Cleaning up...") - - -def after_all(): - """Teardown Sentry""" - print("🧹 Tearing down {SDK Name} tests...") - sentry_sdk.flush(timeout=2.0) - - -def get_mock_sentry_transport(): - """Helper to get mock transport for assertions""" - return get_mock_transport() -``` - -### Step 5: Implement Test Case (e.g., 1-simple.py) - -**CRITICAL: Use the `run_test_case` helper from test_runner.py** - -The framework type is loaded automatically from `config.json`, so you don't need to specify it in test cases. - -```python -""" -1-simple: Basic Completion - -Tests a simple chat completion request with {Your SDK} -and verifies that Sentry captures the appropriate spans and AI monitoring data. -""" - -import os -from your_sdk import YourSDKClient -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - prompt = inputs["prompt"] - - # Your SDK-specific code here - client = YourSDKClient(api_key=os.getenv("YOUR_API_KEY")) - - response = client.generate( - model=model, # model is already overridden via config.json - system=system, - prompt=prompt, - ) - - if not response.text: - raise Exception("No output returned from {Your SDK}") - - # Optional: Log for debugging - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {response.text}") - - return response.text - - -# Framework type is loaded from config.json automatically -test_case = run_test_case("1-simple", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] -``` - -### Step 6: Test Your Implementation - -```bash -cd shared/orchestration -npm run cli -- run --sdk py/{your-sdk} -``` - -## General Testing Commands - -```bash -# Run specific SDK and case -npm run cli run -- --sdk js/your-sdk --case 1-simple - -# Run all cases for an SDK -npm run cli run -- --sdk js/your-sdk - -# Run specific case across all SDKs -npm run cli run -- --case 1-simple - -# Run everything -npm run cli run -- --all -``` - -## See Also - -- [Test Specifications](../shared/specs/README.md) - Fixture format & framework types -- [Test Utilities](../shared/test-utils/README.md) - Mock transport & validation -- [CLI Documentation](../shared/orchestration/README.md) - Test orchestration -- [Troubleshooting](../docs/TROUBLESHOOTING.md) - Common pitfalls -- [Main Documentation](../CLAUDE.md) - Project overview & critical rules diff --git a/sdks/js/_test-utils/README.md b/sdks/js/_test-utils/README.md deleted file mode 100644 index 38c4df1..0000000 --- a/sdks/js/_test-utils/README.md +++ /dev/null @@ -1,198 +0,0 @@ -# JavaScript Test Utilities - -This directory contains JavaScript testing utilities for SDK implementations. - -## Directory Structure - -``` -sdks/js/_test-utils/ -├── test-runner.cjs # Orchestrates test execution (main helper) -├── fixture-loader.cjs # Loads JSON fixtures with config overrides -├── validator.cjs # Validates captured data against fixtures -├── validator.test.cjs # Tests for validator logic -├── mock-transport.cjs # Captures Sentry data in-memory -└── README.md # This file -``` - -**Note:** All files use `.cjs` extension (CommonJS) for compatibility. No package.json - dependencies come from each SDK's package.json. - -## 🚨 CRITICAL: JavaScript/Python Parity Rule - -**The utilities in `sdks/js/_test-utils/` MUST be kept synchronized with `sdks/py/_test-utils/`.** - -### Why This Matters - -- Same fixtures (JSON) used by both languages -- Same validation logic = consistent behavior -- Same error messages = easier debugging -- Changes to one MUST be mirrored in the other - -### When You Change Test Utils - -**ALWAYS update both JS and Python versions together:** - -1. **If you modify `validator.cjs`:** - - Update `sdks/py/_test-utils/validator.py` with equivalent logic - - **Update `validator.test.cjs` and `validator.test.py` with test cases for the new feature** - - Run both test files: `node validator.test.cjs && python3 validator.test.py` - - Run same fixture through both validators - - Confirm identical error output - -2. **If you modify `test-runner.cjs`:** - - Update `sdks/py/_test-utils/test_runner.py` with equivalent logic - - Test both implementations - - Verify error messages match - -3. **If you add a new helper function:** - - Implement in both languages - - Keep function signatures equivalent - - Document any language-specific differences - -## Core Components - -### test-runner.cjs - -The main helper that orchestrates test execution. Provides: - -- **`runTestCase(testCaseId, testLogic, Sentry)`** - Main test orchestration function - - Loads SDK config from environment (`SDK_CONFIG`) - - Loads fixture with `$ref` resolution and config overrides applied - - Displays test name from `fixture.name` (no hardcoded descriptions) - - Wraps test logic in Sentry span - - Validates captured data against fixture - - Shows SDK path in output: `[js/langchain]` - - Returns async function compatible with orchestration runner - -### fixture-loader.cjs - -Loads JSON fixtures from `shared/specs/` with SDK-specific overrides: - -- **`loadFixture(testCaseId, frameworkType, configOverrides)`** - - Reads fixture from `shared/specs/{testCaseId}/fixture-{frameworkType}.json` - - Resolves `$ref` references to shared span definitions (`common-spans.json`) - - Applies config overrides (model names, span attributes) - - Returns fixture with all overrides applied - -**Features:** -- `$ref` syntax: `{ "$ref": "common-spans#/llm_call", "parent": "agent" }` -- Cached common spans for performance -- Override properties merge with referenced spans - -### validator.cjs - -Validates captured Sentry data against fixture expectations: - -- **`validateFixture(testCaseId, spans, transactions, events, frameworkType, configOverrides)`** - - Main validation function that orchestrates all validation steps - - Checks transactions, spans, attributes, hierarchy, events - - Returns validation result with detailed error messages - -**Internal validation functions (not exported):** -- `validateTransactions()`, `validateSpanCounts()`, `validateEvents()` - Count validations -- `validateSpanItems()` - Matches and validates individual spans -- `validateSpanRelationships()` - Validates parent-child hierarchy -- `validateSpanAttributes()` - Validates attributes with schema support -- `normalizeOpToList()`, `formatOpDescription()` - Helper utilities - -**Supported validation features:** -- Wildcard patterns: `"gpt-4*"`, `"*-mini"`, `"*anthropic*"` -- Pattern-based op matching: `{ "pattern": "gen_ai.*", "not": [...] }` -- Schema validation: `{ "type": "json_array", "min_length": 2, "items_have": [...] }`, `{ "type": "plain_string" }` -- Presence checks: `true` (attribute must exist) -- Order-based span matching: Multiple spans with same op matched in fixture order -- `null`/`undefined` treated as missing (not mismatch) - -**Exports:** Only `validateFixture` and `attributeMatches` (for tests) - -**Test file:** `validator.test.cjs` - Run with `node validator.test.cjs` to verify validator logic - -### mock-transport.cjs - -In-memory Sentry transport for testing: - -- **`createMockTransport(options)`** - Factory for mock transport -- **`getMockTransport()`** - Get current transport instance -- **`clearMockTransport()`** - Clear captured data - -## Usage - -### In SDK Setup Files (setup.js) - -```javascript -const Sentry = require("@sentry/node"); -const { config } = require("dotenv"); -const { resolve } = require("path"); -const { createMockTransport } = require("../_test-utils/mock-transport.cjs"); - -// Load environment variables -config({ path: resolve(__dirname, "../../../.env") }); - -// Initialize Sentry with mock transport -// Note: This example uses @sentry/node where AI integrations are auto-enabled. -// If using a different Sentry package (e.g., @sentry/browser), you must use -// manual instrumentation techniques to capture AI agent spans. -Sentry.init({ - dsn: process.env.SENTRY_DSN || "https://public@127.0.0.1/1", - tracesSampleRate: 1.0, - transport: createMockTransport, - sendDefaultPii: true, -}); - -module.exports = { Sentry }; -``` - -### In SDK Config Files (config.json) - -```json -{ - "sdk_name": "your-sdk", - "framework_type": "low-level", - "overrides": { - "1-simple": { - "model": "your-model-name", - "gen_ai.request.model": "your-model-name", - "gen_ai.response.model": "your-model-name" - } - } -} -``` - -### In Test Case Files (cases/1-simple.js) - -```javascript -const { Sentry } = require("../setup"); -const YourSDK = require("your-sdk-package"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, system, prompt } = inputs; - - // Your SDK-specific test logic here - const client = new YourSDK({ apiKey: process.env.YOUR_API_KEY }); - const response = await client.generate({ model, system, prompt }); - - if (!response) { - throw new Error("No completion returned"); - } -} - -// Framework type is loaded from config.json automatically -module.exports = runTestCase("1-simple", testLogic, Sentry); -``` - -## Parity Status - -| Component | JavaScript | Python | Status | Notes | -|-----------|------------|--------|--------|-------| -| Test Runner | `test-runner.cjs` | `test_runner.py` | ✅ Synced | Both orchestrate tests correctly | -| Mock Transport | `mock-transport.cjs` | `mock_transport.py` | ✅ Synced | Both capture envelopes correctly | -| Fixture Loader | `fixture-loader.cjs` | `fixture_loader.py` | ✅ Synced | Both support config overrides | -| Fixture Validator | `validator.cjs` | `validator.py` | ✅ Synced | Schema validation, pattern ops, wildcards | -| Validator Tests | `validator.test.cjs` | `validator.test.py` | ✅ Synced | Both test schema validation | - -## See Also - -- [Python Test Utilities](../../py/_test-utils/README.md) - Python equivalent of these utilities -- [Adding SDKs Guide](../README.md) - Step-by-step SDK implementation guide -- [Test Specifications](../../../shared/specs/README.md) - Fixture format & framework types -- [Main Documentation](../../../CLAUDE.md) - Project overview & critical rules diff --git a/sdks/js/_test-utils/fixture-loader.cjs b/sdks/js/_test-utils/fixture-loader.cjs deleted file mode 100644 index 5879ba5..0000000 --- a/sdks/js/_test-utils/fixture-loader.cjs +++ /dev/null @@ -1,136 +0,0 @@ -/** - * Fixture loader - loads JSON test fixtures from shared/specs/ - */ - -const fs = require("fs"); -const path = require("path"); - -// Cache for common spans -let commonSpansCache = null; - -/** - * Load common span definitions - * - * @returns {Object} Common span definitions - */ -function loadCommonSpans() { - if (commonSpansCache) { - return commonSpansCache; - } - - const commonSpansPath = path.join(__dirname, "../../../shared/specs/common-spans.json"); - - if (!fs.existsSync(commonSpansPath)) { - return {}; - } - - const content = fs.readFileSync(commonSpansPath, "utf-8"); - commonSpansCache = JSON.parse(content); - return commonSpansCache; -} - -/** - * Resolve $ref in a span item - * - * @param {Object} spanItem - Span item that may contain $ref - * @param {Object} commonSpans - Common span definitions - * @returns {Object} Resolved span item - */ -function resolveRef(spanItem, commonSpans) { - if (!spanItem.$ref) { - return spanItem; - } - - // Parse $ref format: "common-spans#/span_name" - const ref = spanItem.$ref; - if (!ref.startsWith("common-spans#/")) { - throw new Error(`Invalid $ref format: ${ref}. Expected format: "common-spans#/span_name"`); - } - - const spanName = ref.substring("common-spans#/".length); - const commonSpan = commonSpans[spanName]; - - if (!commonSpan) { - throw new Error(`Common span not found: ${spanName}`); - } - - // Merge common span with overrides from the reference - // Properties in spanItem (except $ref) override common span properties - const { $ref, ...overrides } = spanItem; - return { ...commonSpan, ...overrides }; -} - -/** - * Apply overrides to a fixture object - * - * @param {Object} fixture - The fixture object to modify - * @param {Object} overrides - Key-value pairs to override in the fixture - * @returns {Object} The modified fixture object - */ -function applyOverrides(fixture, overrides) { - if (!overrides || Object.keys(overrides).length === 0) { - return fixture; - } - - // Deep clone to avoid mutating original - const result = JSON.parse(JSON.stringify(fixture)); - - for (const [key, value] of Object.entries(overrides)) { - // Handle special "model" shorthand - applies to inputs.model - if (key === "model") { - if (result.inputs) { - result.inputs.model = value; - } - continue; - } - - // Handle dot-notation paths in expectations (e.g., "gen_ai.request.model") - // These override values in required_attributes - if (result.expectations && result.expectations.spans && result.expectations.spans.items) { - for (const spanItem of result.expectations.spans.items) { - if (spanItem.required_attributes && key in spanItem.required_attributes) { - spanItem.required_attributes[key] = value; - } - } - } - } - - return result; -} - -/** - * Load a fixture by spec ID and variant - * - * @param {string} specId - The spec ID (e.g., "1-simple", "2-simple-with-error") - * @param {string} variant - The fixture variant (e.g., "agentic", "low-level") - * @param {Object} overrides - Optional key-value overrides to apply to the fixture - * @returns {Object} The parsed fixture object - * @throws {Error} If fixture file not found - */ -function loadFixture(specId, variant = "agentic", overrides = null) { - // Fixtures are in shared/specs/{specId}/fixture-{variant}.json - // Path from sdks/js/_test-utils/ to shared/specs/ - const fixturePath = path.join(__dirname, "../../../shared/specs", specId, `fixture-${variant}.json`); - - if (!fs.existsSync(fixturePath)) { - throw new Error(`Fixture not found: ${specId} (variant: ${variant}) at ${fixturePath}`); - } - - const content = fs.readFileSync(fixturePath, "utf-8"); - const fixture = JSON.parse(content); - - // Resolve $ref references in span items - if (fixture.expectations?.spans?.items) { - const commonSpans = loadCommonSpans(); - fixture.expectations.spans.items = fixture.expectations.spans.items.map(item => - resolveRef(item, commonSpans) - ); - } - - // Apply overrides if provided - return applyOverrides(fixture, overrides); -} - -module.exports = { - loadFixture, -}; diff --git a/sdks/js/_test-utils/mock-transport.cjs b/sdks/js/_test-utils/mock-transport.cjs deleted file mode 100644 index 18c8e9e..0000000 --- a/sdks/js/_test-utils/mock-transport.cjs +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Mock Sentry transport for testing - * - * Captures all Sentry events in memory instead of sending them to Sentry servers. - * Provides helpers to query and verify captured events. - */ - -class MockTransportCapture { - constructor() { - this.envelopes = []; - } - - /** - * Capture an envelope - */ - capture(envelope) { - this.envelopes.push(envelope); - } - - /** - * Clear all captured envelopes - */ - clear() { - this.envelopes = []; - } - - /** - * Get all captured envelopes - */ - getEnvelopes() { - return this.envelopes; - } - - /** - * Parse envelope body string into header and items - * Envelope format is newline-separated: - * Line 1: Envelope headers (JSON) - * Line 2: Item 1 headers (JSON) - * Line 3: Item 1 payload (JSON) - * Line 4: Item 2 headers (JSON) - * Line 5: Item 2 payload (JSON) - * etc. - */ - parseEnvelopeBody(body) { - const lines = body.split("\n").filter((line) => line.trim()); - - if (lines.length === 0) { - return { headers: {}, items: [] }; - } - - // First line is envelope headers - const headers = JSON.parse(lines[0]); - const items = []; - - // Remaining lines are pairs of item headers + item payload - for (let i = 1; i < lines.length; i += 2) { - if (i + 1 < lines.length) { - const itemHeaders = JSON.parse(lines[i]); - const itemPayload = JSON.parse(lines[i + 1]); - items.push({ headers: itemHeaders, payload: itemPayload }); - } - } - - return { headers, items }; - } - - /** - * Get all captured transactions - */ - getTransactions() { - const transactions = []; - - for (const envelope of this.envelopes) { - const body = envelope.body; - if (typeof body !== "string") continue; - - const parsed = this.parseEnvelopeBody(body); - - for (const item of parsed.items) { - if (item.headers.type === "transaction") { - transactions.push(item.payload); - } - } - } - - return transactions; - } - - /** - * Get all captured spans (extracted from transactions) - */ - getSpans() { - const spans = []; - - for (const transaction of this.getTransactions()) { - if (transaction.spans && Array.isArray(transaction.spans)) { - spans.push(...transaction.spans); - } - } - - return spans; - } - - /** - * Get all captured events (errors, messages, etc.) - */ - getEvents() { - const events = []; - - for (const envelope of this.envelopes) { - const body = envelope.body; - if (typeof body !== "string") continue; - - const parsed = this.parseEnvelopeBody(body); - - for (const item of parsed.items) { - if (item.headers.type === "event") { - events.push(item.payload); - } - } - } - - return events; - } -} - -let mockTransportCapture = null; - -/** - * Create a mock transport factory (to be passed to Sentry.init) - * - * @param {Function} createTransport - The createTransport function from @sentry/core - * @returns {Function} Transport factory function - */ -function createMockTransport(createTransport) { - return function(options) { - // Initialize capture instance - mockTransportCapture = new MockTransportCapture(); - - // Create transport using Sentry's createTransport helper - return createTransport(options, (envelope) => { - // Capture the envelope - if (mockTransportCapture) { - mockTransportCapture.capture(envelope); - } - - // Return success response - return Promise.resolve({ - statusCode: 200, - headers: {}, - }); - }); - }; -} - -/** - * Get the current mock transport capture instance - */ -function getMockTransport() { - if (!mockTransportCapture) { - throw new Error( - "Mock transport not initialized. Did you call Sentry.init with createMockTransport?" - ); - } - return mockTransportCapture; -} - -/** - * Clear all captured events - */ -function clearMockTransport() { - if (mockTransportCapture) { - mockTransportCapture.clear(); - } -} - -module.exports = { - createMockTransport, - getMockTransport, - clearMockTransport, -}; diff --git a/sdks/js/_test-utils/test-runner.cjs b/sdks/js/_test-utils/test-runner.cjs deleted file mode 100644 index 2c3a71a..0000000 --- a/sdks/js/_test-utils/test-runner.cjs +++ /dev/null @@ -1,91 +0,0 @@ -/** - * Test runner helper - orchestrates test execution with minimal boilerplate - */ - -const { loadFixture } = require("./fixture-loader.cjs"); -const { validateFixture } = require("./validator.cjs"); -const { getMockTransport } = require("./mock-transport.cjs"); - -/** - * Run a test case with automatic fixture loading, span wrapping, and validation - * - * @param {string} specId - Test spec ID (e.g., "1-simple") - * @param {Function} testLogic - Async function containing SDK-specific test logic - * @param {Object} Sentry - Sentry SDK instance - * @returns {Function} Test function ready to be exported - */ -function runTestCase(specId, testLogic, Sentry) { - return async function () { - const sdkPath = process.env.SDK_PATH || 'unknown'; - - // Load SDK config from environment - const sdkConfig = process.env.SDK_CONFIG - ? JSON.parse(process.env.SDK_CONFIG) - : null; - - if (!sdkConfig?.framework_type) { - throw new Error( - "SDK_CONFIG with framework_type must be provided via environment variable" - ); - } - - const frameworkType = sdkConfig.framework_type; - - // Load config overrides from environment - const overrides = process.env.SDK_CONFIG_OVERRIDES - ? JSON.parse(process.env.SDK_CONFIG_OVERRIDES) - : null; - - // Load fixture inputs with overrides applied - const fixture = loadFixture(specId, frameworkType, overrides); - - // Log with test name from fixture - console.log(`\n [${sdkPath}]`); - console.log(` Running ${specId}: ${fixture.name || specId}`); - - // Create main span for this test - await Sentry.startSpan({ name: `${specId}-test`, op: "test" }, async () => { - await testLogic(fixture.inputs); - }); - - // Flush Sentry to ensure all events are sent to transport - await Sentry.flush(2000); - - // Small buffer to ensure transport has processed everything - await new Promise((resolve) => setTimeout(resolve, 50)); - - // Verify Sentry captured the expected data - const transport = getMockTransport(); - const spans = transport.getSpans(); - const transactions = transport.getTransactions(); - const events = transport.getEvents(); - - console.log( - ` Captured: ${spans.length} spans, ${transactions.length} transactions, ${events.length} events` - ); - - const result = validateFixture( - specId, - spans, - transactions, - events, - frameworkType, - overrides - ); - - if (!result.passed) { - console.log(" ✗ Validation failed:"); - result.errors.forEach((error) => console.log(` - ${error}`)); - throw new Error( - `Fixture validation failed:\n${result.errors.join("\n")}` - ); - } - - console.log(" ✓ All fixture validations passed"); - console.log(` ✓ ${specId} completed`); - }; -} - -module.exports = { - runTestCase, -}; diff --git a/sdks/js/_test-utils/validator.cjs b/sdks/js/_test-utils/validator.cjs deleted file mode 100644 index 80fab01..0000000 --- a/sdks/js/_test-utils/validator.cjs +++ /dev/null @@ -1,850 +0,0 @@ -/** - * Fixture validator - validates captured Sentry data against fixtures - * - * Includes assertion helpers for querying and verifying spans - */ - -const { loadFixture } = require("./fixture-loader.cjs"); - -// ============================================================================ -// ASSERTION HELPERS -// ============================================================================ - -/** - * Format an op specification as a human-readable description - * - * @param {string|Array|Object} op - Operation specification - * @returns {string} Human-readable description - */ -function formatOpDescription(op) { - if (typeof op === "object" && !Array.isArray(op)) { - return `${op.pattern} (excluding: ${(op.not || []).join(", ")})`; - } - return Array.isArray(op) ? op.join(" or ") : op; -} - -/** - * Normalize an op specification to a list of operation names - * - * @param {string|Array|Object} op - Operation specification - * @param {Array} spans - Available spans (needed for pattern matching) - * @returns {Array} List of operation names - */ -function normalizeOpToList(op, spans) { - if (typeof op === "object" && !Array.isArray(op)) { - // Object format: { pattern: "gen_ai.*", not: ["gen_ai.invoke_agent", ...] } - const pattern = op.pattern; - const notList = op.not || []; - - // Get all unique op values from spans that match the pattern but not in the exclusion list - const matchingOps = new Set(); - spans.forEach((s) => { - if (s.op && matchesPattern(s.op, pattern) && !notList.includes(s.op)) { - matchingOps.add(s.op); - } - }); - - return Array.from(matchingOps); - } - - // String or array format - return Array.isArray(op) ? op : [op]; -} - -/** - * Validate span attributes and collect errors - * - * @param {Object} span - The span to validate - * @param {Object} requiredAttributes - Required attributes to check - * @returns {Object} Object with {missing: [], mismatched: []} arrays - */ -function validateSpanAttributes(span, requiredAttributes) { - const errors = { missing: [], mismatched: [] }; - - for (const [attr, expectedValue] of Object.entries(requiredAttributes)) { - if (expectedValue === true) { - // Just check presence - if (!hasAttribute(span, attr)) { - errors.missing.push(attr); - } - } else { - // Check value matches - if (!attributeMatches(span, attr, expectedValue)) { - const actualValue = getAttribute(span, attr); - // Treat undefined/null as missing, not mismatch - if (actualValue === undefined || actualValue === null) { - errors.missing.push(attr); - } else { - errors.mismatched.push({ - attr, - expected: expectedValue, - actual: actualValue, - }); - } - } - } - } - - return errors; -} - -/** - * Get a single span by operation name(s) and/or attributes - * Throws if zero or more than one span is found - * - * @param {Array} spans - Array of span objects - * @param {string|Array|Object} op - Operation name (string), array of operation names, or object with pattern/not - * @param {Object} [requiredAttributes] - Optional object of attributes to filter by - * @param {Set} [usedSpans] - Set of span IDs already used (for matching multiple spans in order) - * @returns {Object} The matching span - * @throws {Error} If zero or multiple spans found - */ -function getSpan(spans, op, requiredAttributes, usedSpans) { - // Normalize op to a list of operation names - const opList = normalizeOpToList(op, spans); - - // Filter by operation name(s) - let matching = spans.filter((s) => opList.includes(s.op)); - - // Exclude already-used spans if usedSpans is provided - if (usedSpans) { - matching = matching.filter((s) => !usedSpans.has(s.span_id)); - } - - // Further filter by attributes if specified - if (requiredAttributes) { - matching = matching.filter((s) => - containsAttributes(s, requiredAttributes) - ); - } - - if (matching.length === 0) { - const opDesc = formatOpDescription(op); - - // Check if any spans with the op exist (without attribute filtering) - const spansWithOp = spans.filter((s) => opList.includes(s.op)); - - if (spansWithOp.length === 0) { - // No spans with that op at all - let errorMsg = `No span found with op="${opDesc}"`; - errorMsg += `\n Available spans:`; - spans.forEach((s, i) => { - errorMsg += `\n ${i + 1}. op="${s.op}"`; - }); - throw new Error(errorMsg); - } else { - // Spans with that op exist, but don't match required attributes - const isVerbose = process.env.SENTRY_AI_TEST_VERBOSE === "true"; - let errorMsg = `Found span with op="${opDesc}" but missing required attributes`; - - if (requiredAttributes) { - const span = spansWithOp[0]; - const spanData = span.data || {}; - - // Concise mode: Just show what's missing/mismatched - if (!isVerbose) { - const missing = []; - const mismatched = []; - - for (const [attr, expectedVal] of Object.entries( - requiredAttributes - )) { - const actualVal = getAttribute(span, attr); - - if (actualVal === undefined) { - missing.push(attr); - } else if (expectedVal !== true && actualVal !== expectedVal) { - mismatched.push( - `${attr} (expected: ${JSON.stringify( - expectedVal - )}, got: ${JSON.stringify(actualVal)})` - ); - } - } - - if (missing.length > 0) { - errorMsg += `\n Missing: ${missing.join(", ")}`; - } - if (mismatched.length > 0) { - errorMsg += `\n Mismatched: ${mismatched.join(", ")}`; - } - errorMsg += `\n (run with --verbose for full details)`; - } else { - // Verbose mode: Show everything - errorMsg += `\n Required attributes:`; - for (const [attr, val] of Object.entries(requiredAttributes)) { - errorMsg += `\n - ${attr}: ${ - val === true ? "(any value)" : JSON.stringify(val) - }`; - } - - errorMsg += `\n Span's actual attributes:`; - const dataKeys = Object.keys(spanData); - if (dataKeys.length > 0) { - dataKeys.forEach((key) => { - errorMsg += `\n - ${key}: ${JSON.stringify(spanData[key])}`; - }); - } else { - errorMsg += `\n (no attributes)`; - } - } - } - - throw new Error(errorMsg); - } - } - - if (matching.length > 1) { - // If usedSpans is provided, we're matching in order - return first match - if (usedSpans) { - return matching[0]; - } - - // Otherwise, multiple matches is an error - const opDesc = formatOpDescription(op); - let errorMsg = `Found ${matching.length} spans matching op="${opDesc}", expected exactly 1`; - - errorMsg += `\n Matching spans:`; - matching.forEach((s, i) => { - errorMsg += `\n ${i + 1}. op="${s.op}" span_id=${( - s.span_id || "?" - ).substring(0, 8)}`; - }); - - throw new Error(errorMsg); - } - - return matching[0]; -} - -/** - * Get all spans matching an operation name - * - * @param {Array} spans - Array of span objects - * @param {string} op - Operation name to search for - * @returns {Array} Array of matching spans (may be empty) - */ -function getSpans(spans, op) { - return spans.filter((s) => s.op === op); -} - -/** - * Check if one span is a child of another - * - * @param {Object} childSpan - The potential child span - * @param {Object} parentSpan - The potential parent span - * @returns {boolean} True if childSpan is a child of parentSpan - */ -function isChildOf(childSpan, parentSpan) { - if (!childSpan || !parentSpan) { - return false; - } - - // Check if child's parent_span_id matches parent's span_id - return childSpan.parent_span_id === parentSpan.span_id; -} - -/** - * Get an attribute value from a span - * First checks span.data for the attribute, then checks span directly - * - * @param {Object} span - The span to check - * @param {string} attributeName - Name of the attribute (e.g., "gen_ai.request.model") - * @returns {*} The attribute value, or undefined if not found - */ -function getAttribute(span, attributeName) { - if (!span) { - return undefined; - } - - // First check in span.data (where Sentry stores span attributes) - if ( - span.data && - typeof span.data === "object" && - attributeName in span.data - ) { - return span.data[attributeName]; - } - - // Then check directly on span using dot notation - const parts = attributeName.split("."); - let current = span; - - for (const part of parts) { - if (current && typeof current === "object" && part in current) { - current = current[part]; - } else { - return undefined; - } - } - - return current; -} - -/** - * Check if a span has an attribute (regardless of value) - * - * @param {Object} span - The span to check - * @param {string} attributeName - Name of the attribute (can use dot notation for nested, e.g., "data.ai.model") - * @returns {boolean} True if attribute exists - */ -function hasAttribute(span, attributeName) { - return getAttribute(span, attributeName) !== undefined; -} - -/** - * Check if a value matches a pattern with wildcard support - * - * Wildcard patterns: - * - "foo*" matches any string that begins with "foo" - * - "*foo" matches any string that ends with "foo" - * - "*foo*" matches any string that contains "foo" - * - "foo" matches exactly "foo" (no wildcards) - * - * @param {*} actualValue - The actual value to test - * @param {*} pattern - The expected value or pattern (may contain wildcards) - * @returns {boolean} True if the value matches the pattern - */ -function matchesPattern(actualValue, pattern) { - // If pattern is not a string, use strict equality - if (typeof pattern !== "string") { - return actualValue === pattern; - } - - // Convert actual value to string for pattern matching - const actualStr = String(actualValue); - - // Check for wildcard patterns - if (pattern.includes("*")) { - // *foo* - contains - if (pattern.startsWith("*") && pattern.endsWith("*")) { - const substring = pattern.slice(1, -1); - // If substring is empty (pattern is "*" or "**"), no match - if (substring === "" || substring === "*") { - return false; - } - return actualStr.includes(substring); - } - // foo* - starts with - else if (pattern.endsWith("*")) { - const prefix = pattern.slice(0, -1); - // If prefix is empty (pattern is just "*"), no match - if (prefix === "") { - return false; - } - return actualStr.startsWith(prefix); - } - // *foo - ends with - else if (pattern.startsWith("*")) { - const suffix = pattern.slice(1); - // If suffix is empty (pattern is just "*"), no match - if (suffix === "") { - return false; - } - return actualStr.endsWith(suffix); - } - } - - // No wildcards, use strict equality - return actualValue === pattern; -} - -/** - * Validate an attribute against a schema object - * - * @param {*} attrValue - The actual attribute value - * @param {Object} schema - Schema object with validation rules - * @param {Object} span - The span object (needed for cross-attribute constraints like lte) - * @returns {boolean} True if value matches schema - * - * Supported schema formats: - * - { type: "json_array", min_length: 2, items_have: ["role", "content"] } - * - { type: "json_array", length: 2, items_have: ["role"] } - * - { type: "plain_string", min_length: 1, pattern: "*hello*" } - * - { type: "number", lte: "other.attribute.name" } - value must be <= other attribute - */ -function validateSchema(attrValue, schema, span = null) { - if (!schema || typeof schema !== "object") { - return false; - } - - // Handle plain_string type - if (schema.type === "plain_string") { - // Must be a string - if (typeof attrValue !== "string") { - return false; - } - - // Must NOT be valid JSON - try { - JSON.parse(attrValue); - return false; // It's valid JSON, so it's not a plain string - } catch (e) { - // Good - not JSON, it's a plain string - } - - // Validate min_length - if ( - schema.min_length !== undefined && - attrValue.length < schema.min_length - ) { - return false; - } - - // Validate max_length - if ( - schema.max_length !== undefined && - attrValue.length > schema.max_length - ) { - return false; - } - - // Validate pattern - if ( - schema.pattern !== undefined && - !matchesPattern(attrValue, schema.pattern) - ) { - return false; - } - - return true; - } - - // Handle json_array type - if (schema.type === "json_array") { - // Store the original string for 'contains' check - const originalString = typeof attrValue === "string" ? attrValue : JSON.stringify(attrValue); - - // Parse JSON if it's a string - let parsed; - if (typeof attrValue === "string") { - try { - parsed = JSON.parse(attrValue); - } catch (e) { - return false; // Not valid JSON - } - } else { - parsed = attrValue; - } - - // Check if it's an array - if (!Array.isArray(parsed)) { - return false; - } - - // Validate length - if (schema.length !== undefined && parsed.length !== schema.length) { - return false; - } - - if (schema.min_length !== undefined && parsed.length < schema.min_length) { - return false; - } - - if (schema.max_length !== undefined && parsed.length > schema.max_length) { - return false; - } - - // Validate length_lte (array length must be <= another attribute's value) - if (schema.length_lte !== undefined && span !== null) { - const otherValue = getAttribute(span, schema.length_lte); - // Only validate if the other attribute exists and is a number - if (otherValue !== undefined && typeof otherValue === "number") { - if (parsed.length > otherValue) { - return false; - } - } - } - - // Validate contains (raw JSON string must contain this substring) - if (schema.contains !== undefined) { - if (!originalString.includes(schema.contains)) { - return false; - } - } - - // Validate items_have (each item must have these properties) - if (schema.items_have && Array.isArray(schema.items_have)) { - for (const item of parsed) { - if (typeof item !== "object" || item === null) { - return false; - } - for (const requiredProp of schema.items_have) { - if (!(requiredProp in item)) { - return false; - } - } - } - } - - return true; - } - - // Handle number type with constraints - if (schema.type === "number") { - // Must be a number - if (typeof attrValue !== "number") { - return false; - } - - // Validate lte (less than or equal to another attribute) - if (schema.lte !== undefined && span !== null) { - const otherValue = getAttribute(span, schema.lte); - // Only validate if the other attribute exists and is a number - if (otherValue !== undefined && typeof otherValue === "number") { - if (attrValue > otherValue) { - return false; - } - } - } - - return true; - } - - // Unknown schema type - return false; -} - -/** - * Check if a span has an attribute with a specific value - * - * @param {Object} span - The span to check - * @param {string} attributeName - Name of the attribute (can use dot notation for nested, e.g., "data.ai.model") - * @param {*} value - Expected value (uses strict equality, wildcard pattern, or schema validation) - * @returns {boolean} True if attribute exists and matches value - */ -function attributeMatches(span, attributeName, value) { - const attrValue = getAttribute(span, attributeName); - - // Check if value is a schema object with optional flag - if ( - typeof value === "object" && - value !== null && - !Array.isArray(value) && - value.type - ) { - // If attribute is missing and schema marks it as optional, that's OK - if (attrValue === undefined && value.optional === true) { - return true; - } - if (attrValue === undefined) { - return false; - } - return validateSchema(attrValue, value, span); - } - - // For non-schema values, missing attribute means no match - if (attrValue === undefined) { - return false; - } - - // Otherwise use pattern matching - return matchesPattern(attrValue, value); -} - -/** - * Check if a span contains multiple attributes - * - * @param {Object} span - The span to check - * @param {Object} attributes - Object mapping attribute names to expected values - * - If value is `true`, only checks attribute presence - * - Otherwise checks attribute matches the value exactly - * @returns {boolean} True if all attributes match - * - * @example - * assert(containsAttributes(span, { - * 'gen_ai.request.model': 'gpt-5-nano', // checks value matches - * 'gen_ai.response.text': true, // just checks presence - * 'gen_ai.usage.input_tokens': true, - * })); - */ -function containsAttributes(span, attributes) { - for (const [attrName, expectedValue] of Object.entries(attributes)) { - if (expectedValue === true) { - // Just check presence - if (!hasAttribute(span, attrName)) { - return false; - } - } else { - // Check value matches - if (!attributeMatches(span, attrName, expectedValue)) { - return false; - } - } - } - - return true; -} - -// ============================================================================ -// VALIDATOR -// ============================================================================ - -/** - * Validate transaction count - * - * @param {Array} transactions - Captured transactions - * @param {Object} expectations - Fixture expectations - * @param {Array} errors - Error array to append to - */ -function validateTransactions(transactions, expectations, errors) { - if (expectations.transactions) { - const { min_count } = expectations.transactions; - if (min_count !== undefined && transactions.length < min_count) { - errors.push( - `Expected at least ${min_count} transaction(s), got ${transactions.length}` - ); - } - } -} - -/** - * Validate span count - * - * @param {Array} spans - Captured spans - * @param {Object} expectations - Fixture span expectations - * @param {Array} errors - Error array to append to - */ -function validateSpanCounts(spans, expectations, errors) { - if (expectations.spans) { - const { count, min_count } = expectations.spans; - const minSpanCount = min_count !== undefined ? min_count : count; - if (minSpanCount !== undefined && spans.length < minSpanCount) { - errors.push(`Expected at least ${minSpanCount} span(s), got ${spans.length}`); - } - } -} - -/** - * Validate events - * - * @param {Array} events - Captured events - * @param {Object} expectations - Fixture expectations - * @param {Array} errors - Error array to append to - */ -function validateEvents(events, expectations, errors) { - if (expectations.events) { - const { error_count } = expectations.events; - if (error_count !== undefined) { - const actualErrorCount = events.filter((e) => e.level === "error").length; - if (actualErrorCount !== error_count) { - errors.push(`Expected ${error_count} error event(s), got ${actualErrorCount}`); - } - } - } -} - -/** - * Validate parent-child relationships between spans - * - * @param {Array} items - Span item expectations from fixture - * @param {Map} spanMap - Map of fixture ID to matched span - * @param {Array} errors - Error array to append to - */ -function validateSpanRelationships(items, spanMap, errors) { - for (const itemExpectation of items) { - if (itemExpectation.parent) { - const childSpan = spanMap.get(itemExpectation.id); - const parentSpan = spanMap.get(itemExpectation.parent); - - if (childSpan && parentSpan) { - if (!isChildOf(childSpan, parentSpan)) { - errors.push( - `Span with op="${formatOpDescription(itemExpectation.op)}" should be child of span with id="${itemExpectation.parent}"` - ); - } - } - } - } -} - -/** - * Validate individual span items from fixture expectations - * - * @param {Array} spans - Captured spans - * @param {Array} items - Span item expectations from fixture - * @param {Array} errors - Error array to append to - * @returns {Map} Map of fixture ID to matched span - */ -function validateSpanItems(spans, items, errors) { - const spanMap = new Map(); - const spanErrors = new Map(); - const usedSpans = new Set(); - - // Match each expected span - for (const itemExpectation of items) { - const fixtureId = itemExpectation.id; - const expectedOp = formatOpDescription(itemExpectation.op); - - try { - // Get span by operation and attributes (pass usedSpans to match in order) - const requiredAttrs = itemExpectation.required_attributes; - const span = getSpan(spans, itemExpectation.op, requiredAttrs, usedSpans); - spanMap.set(fixtureId, span); - - // Mark this span as used - if (span.span_id) { - usedSpans.add(span.span_id); - } - - // Validate required attributes and collect errors - if (requiredAttrs) { - if (!spanErrors.has(fixtureId)) { - spanErrors.set(fixtureId, { - expectedOp, - actualOp: span.op, - missing: [], - mismatched: [], - }); - } - const spanError = spanErrors.get(fixtureId); - - // Validate attributes and collect errors - const attrErrors = validateSpanAttributes(span, requiredAttrs); - spanError.missing.push(...attrErrors.missing); - spanError.mismatched.push(...attrErrors.mismatched); - } - } catch (error) { - // getSpan threw an error - check if it's about missing attributes or missing span - if (error.message.includes("but missing required attributes")) { - // Span exists but has attribute issues - extract the details - const requiredAttrs = itemExpectation.required_attributes; - if (requiredAttrs) { - // Find the span by op only (without attribute filtering) - const opList = normalizeOpToList(itemExpectation.op, spans); - const matchingSpan = spans.find((s) => opList.includes(s.op)); - - if (matchingSpan) { - if (!spanErrors.has(fixtureId)) { - spanErrors.set(fixtureId, { - expectedOp, - actualOp: matchingSpan.op, - missing: [], - mismatched: [], - }); - } - const spanError = spanErrors.get(fixtureId); - - // Validate attributes and collect errors - const attrErrors = validateSpanAttributes(matchingSpan, requiredAttrs); - spanError.missing.push(...attrErrors.missing); - spanError.mismatched.push(...attrErrors.mismatched); - } - } - } else if (error.message.includes("No span found with op=")) { - // Span doesn't exist at all - if (!spanErrors.has(fixtureId)) { - spanErrors.set(fixtureId, { - expectedOp, - actualOp: null, - missing: [], - mismatched: [], - notFound: true, - }); - } - } else { - // Other error - just append it - errors.push(error.message); - } - } - } - - // Format span errors in a structured way - for (const [fixtureId, errorDetails] of spanErrors) { - if (errorDetails.notFound) { - errors.push( - ` ${fixtureId} (expected: ${errorDetails.expectedOp}): span not found` - ); - } else if ( - errorDetails.missing.length > 0 || - errorDetails.mismatched.length > 0 - ) { - let errorMsg = ` ${fixtureId} (${errorDetails.actualOp}):`; - - for (const attr of errorDetails.missing) { - errorMsg += `\n ${attr}: missing`; - } - - for (const mismatch of errorDetails.mismatched) { - errorMsg += `\n ${ - mismatch.attr - }: mismatch (expected: ${JSON.stringify( - mismatch.expected - )}, got: ${JSON.stringify(mismatch.actual)})`; - } - - errors.push(errorMsg); - } - } - - return spanMap; -} - -/** - * Validate captured Sentry data against a fixture - * - * @param {string} specId - The spec ID (e.g., "1-simple") - * @param {Array} spans - Captured spans - * @param {Array} transactions - Captured transactions - * @param {Array} events - Captured events (optional) - * @param {string} variant - The fixture variant (e.g., "agentic", "low-level") - * @param {Object} overrides - Optional SDK config overrides to apply to fixture expectations - * @returns {Object} Validation result with { passed, errors } - */ -function validateFixture( - specId, - spans, - transactions, - events = [], - variant = "agentic", - overrides = null -) { - const fixture = loadFixture(specId, variant, overrides); - const errors = []; - - // Log captured spans in verbose mode - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log("\n === Captured Spans (Verbose) ==="); - if (spans.length === 0) { - console.log(" No spans captured"); - } else { - spans.forEach((span, index) => { - console.log(` Span ${index + 1}:`); - console.log(` op: ${span.op || "N/A"}`); - console.log(` description: ${span.description || "N/A"}`); - console.log(` span_id: ${span.span_id || "N/A"}`); - console.log(` parent_span_id: ${span.parent_span_id || "N/A"}`); - if (span.data && Object.keys(span.data).length > 0) { - console.log(` data keys: ${Object.keys(span.data).join(", ")}`); - } - }); - } - console.log(" === End Captured Spans ===\n"); - } - - // Validate transactions - validateTransactions(transactions, fixture.expectations, errors); - - // Validate span counts - validateSpanCounts(spans, fixture.expectations, errors); - - // Validate individual spans and relationships - if (fixture.expectations.spans?.items) { - const spanMap = validateSpanItems(spans, fixture.expectations.spans.items, errors); - validateSpanRelationships(fixture.expectations.spans.items, spanMap, errors); - } - - // Validate events - validateEvents(events, fixture.expectations, errors); - - return { - passed: errors.length === 0, - errors, - fixture, - }; -} - -module.exports = { - validateFixture, - attributeMatches, // Used by validator.test.cjs -}; diff --git a/sdks/js/_test-utils/validator.test.cjs b/sdks/js/_test-utils/validator.test.cjs deleted file mode 100644 index d0d40c1..0000000 --- a/sdks/js/_test-utils/validator.test.cjs +++ /dev/null @@ -1,701 +0,0 @@ -/** - * Simple test for validator - verifies schema validation works - */ - -const { attributeMatches } = require('./validator.cjs'); - -console.log('\n=== JS Validator Schema Tests ===\n'); - -let passed = 0; -let failed = 0; - -// Test 1: JSON array with correct length -{ - const span = { - data: { - 'gen_ai.request.messages': JSON.stringify([ - { role: 'system', content: 'Hello' }, - { role: 'user', content: 'Hi' } - ]) - } - }; - - const schema = { type: 'json_array', length: 2, items_have: ['role', 'content'] }; - const result = attributeMatches(span, 'gen_ai.request.messages', schema); - - if (result === true) { - console.log('✓ Test 1: Exact length match (2 == 2)'); - passed++; - } else { - console.log('✗ Test 1: FAILED - Should match length 2'); - failed++; - } -} - -// Test 2: JSON array with insufficient length -{ - const span = { - data: { - 'gen_ai.request.messages': JSON.stringify([ - { role: 'user', content: 'Hi' } - ]) - } - }; - - const schema = { type: 'json_array', min_length: 2, items_have: ['role'] }; - const result = attributeMatches(span, 'gen_ai.request.messages', schema); - - if (result === false) { - console.log('✓ Test 2: Min length violation (1 < 2)'); - passed++; - } else { - console.log('✗ Test 2: FAILED - Should reject length < min_length'); - failed++; - } -} - -// Test 3: Missing required property in items -{ - const span = { - data: { - 'gen_ai.request.messages': JSON.stringify([ - { role: 'user' }, // Missing 'content' - { role: 'system', content: 'Hi' } - ]) - } - }; - - const schema = { type: 'json_array', items_have: ['role', 'content'] }; - const result = attributeMatches(span, 'gen_ai.request.messages', schema); - - if (result === false) { - console.log('✓ Test 3: Missing required property in item'); - passed++; - } else { - console.log('✗ Test 3: FAILED - Should detect missing "content" property'); - failed++; - } -} - -// Test 4: Regular string matching (backward compatibility) -{ - const span = { data: { 'gen_ai.request.model': 'gpt-4' } }; - const result = attributeMatches(span, 'gen_ai.request.model', 'gpt-4'); - - if (result === true) { - console.log('✓ Test 4: Regular string matching still works'); - passed++; - } else { - console.log('✗ Test 4: FAILED - String matching broken'); - failed++; - } -} - -// Test 5: Wildcard pattern matching (backward compatibility) -{ - const span = { data: { 'gen_ai.response.model': 'gpt-4-turbo-preview' } }; - const result = attributeMatches(span, 'gen_ai.response.model', 'gpt-4*'); - - if (result === true) { - console.log('✓ Test 5: Wildcard pattern matching still works'); - passed++; - } else { - console.log('✗ Test 5: FAILED - Wildcard matching broken'); - failed++; - } -} - -// Test 6: Plain string (not JSON) -{ - const span = { data: { 'gen_ai.response.text': 'Hello world' } }; - const schema = { type: 'plain_string', min_length: 1 }; - const result = attributeMatches(span, 'gen_ai.response.text', schema); - - if (result === true) { - console.log('✓ Test 6: Plain string validation passes'); - passed++; - } else { - console.log('✗ Test 6: FAILED - Plain string should pass'); - failed++; - } -} - -// Test 7: JSON string rejected as plain_string -{ - const span = { data: { 'gen_ai.response.text': '[1, 2, 3]' } }; - const schema = { type: 'plain_string' }; - const result = attributeMatches(span, 'gen_ai.response.text', schema); - - if (result === false) { - console.log('✓ Test 7: JSON string rejected as plain_string'); - passed++; - } else { - console.log('✗ Test 7: FAILED - Should reject JSON strings'); - failed++; - } -} - -// Test 8: Plain string with pattern -{ - const span = { data: { 'gen_ai.system': 'You are a helpful assistant' } }; - const schema = { type: 'plain_string', pattern: '*helpful*' }; - const result = attributeMatches(span, 'gen_ai.system', schema); - - if (result === true) { - console.log('✓ Test 8: Plain string with pattern match'); - passed++; - } else { - console.log('✗ Test 8: FAILED - Pattern should match'); - failed++; - } -} - -// Test 9: Plain string min_length violation -{ - const span = { data: { 'gen_ai.response.text': 'Hi' } }; - const schema = { type: 'plain_string', min_length: 10 }; - const result = attributeMatches(span, 'gen_ai.response.text', schema); - - if (result === false) { - console.log('✓ Test 9: Plain string min_length violation (2 < 10)'); - passed++; - } else { - console.log('✗ Test 9: FAILED - Should reject string shorter than min_length'); - failed++; - } -} - -// ============================================================================ -// Number Schema with lte Constraint Tests -// ============================================================================ - -console.log('\n--- Number Schema with lte Constraint Tests ---\n'); - -// Test 10: Valid lte constraint (50 <= 100) -{ - const span = { - data: { - 'gen_ai.usage.input_tokens': 100, - 'gen_ai.usage.input_tokens.cached': 50 - } - }; - const schema = { type: 'number', lte: 'gen_ai.usage.input_tokens' }; - const result = attributeMatches(span, 'gen_ai.usage.input_tokens.cached', schema); - - if (result === true) { - console.log('✓ Test 10: Valid lte constraint (50 <= 100)'); - passed++; - } else { - console.log('✗ Test 10: FAILED - Should pass when value <= other'); - failed++; - } -} - -// Test 11: Invalid lte constraint (100 > 50) -{ - const span = { - data: { - 'gen_ai.usage.input_tokens': 50, - 'gen_ai.usage.input_tokens.cached': 100 - } - }; - const schema = { type: 'number', lte: 'gen_ai.usage.input_tokens' }; - const result = attributeMatches(span, 'gen_ai.usage.input_tokens.cached', schema); - - if (result === false) { - console.log('✓ Test 11: Invalid lte constraint detected (100 > 50)'); - passed++; - } else { - console.log('✗ Test 11: FAILED - Should fail when value > other'); - failed++; - } -} - -// Test 12: Edge case - lte with equal values (100 <= 100) -{ - const span = { - data: { - 'gen_ai.usage.input_tokens': 100, - 'gen_ai.usage.input_tokens.cached': 100 - } - }; - const schema = { type: 'number', lte: 'gen_ai.usage.input_tokens' }; - const result = attributeMatches(span, 'gen_ai.usage.input_tokens.cached', schema); - - if (result === true) { - console.log('✓ Test 12: Edge case - lte with equal values (100 <= 100)'); - passed++; - } else { - console.log('✗ Test 12: FAILED - Should pass when value == other'); - failed++; - } -} - -// Test 13: lte constraint passes when other attribute is missing -{ - const span = { - data: { - 'gen_ai.usage.input_tokens.cached': 100 - } - }; - const schema = { type: 'number', lte: 'gen_ai.usage.input_tokens' }; - const result = attributeMatches(span, 'gen_ai.usage.input_tokens.cached', schema); - - if (result === true) { - console.log('✓ Test 13: lte passes when other attribute is missing'); - passed++; - } else { - console.log('✗ Test 13: FAILED - Should pass when other attribute is missing'); - failed++; - } -} - -// Test 14: Number schema fails for non-number values -{ - const span = { - data: { - 'gen_ai.usage.input_tokens': 100, - 'gen_ai.usage.input_tokens.cached': 'not a number' - } - }; - const schema = { type: 'number', lte: 'gen_ai.usage.input_tokens' }; - const result = attributeMatches(span, 'gen_ai.usage.input_tokens.cached', schema); - - if (result === false) { - console.log('✓ Test 14: Number schema fails for non-number values'); - passed++; - } else { - console.log('✗ Test 14: FAILED - Should fail for non-number values'); - failed++; - } -} - -// Test 15: Number schema without lte constraint (just type check) -{ - const span = { - data: { - 'gen_ai.usage.input_tokens': 100 - } - }; - const schema = { type: 'number' }; - const result = attributeMatches(span, 'gen_ai.usage.input_tokens', schema); - - if (result === true) { - console.log('✓ Test 15: Number schema without lte constraint passes'); - passed++; - } else { - console.log('✗ Test 15: FAILED - Should pass when only checking type'); - failed++; - } -} - -// Test 16: Valid lte for output tokens reasoning (150 <= 200) -{ - const span = { - data: { - 'gen_ai.usage.output_tokens': 200, - 'gen_ai.usage.output_tokens.reasoning': 150 - } - }; - const schema = { type: 'number', lte: 'gen_ai.usage.output_tokens' }; - const result = attributeMatches(span, 'gen_ai.usage.output_tokens.reasoning', schema); - - if (result === true) { - console.log('✓ Test 16: Valid lte for output tokens reasoning (150 <= 200)'); - passed++; - } else { - console.log('✗ Test 16: FAILED - Should pass when value <= other'); - failed++; - } -} - -// Test 17: Invalid lte for output tokens reasoning (150 > 100) -{ - const span = { - data: { - 'gen_ai.usage.output_tokens': 100, - 'gen_ai.usage.output_tokens.reasoning': 150 - } - }; - const schema = { type: 'number', lte: 'gen_ai.usage.output_tokens' }; - const result = attributeMatches(span, 'gen_ai.usage.output_tokens.reasoning', schema); - - if (result === false) { - console.log('✓ Test 17: Invalid lte for output tokens reasoning detected (150 > 100)'); - passed++; - } else { - console.log('✗ Test 17: FAILED - Should fail when value > other'); - failed++; - } -} - -// ============================================================================ -// Optional Schema Attribute Tests -// ============================================================================ - -console.log('\n--- Optional Schema Attribute Tests ---\n'); - -// Test 18: Optional attribute missing - should pass -{ - const span = { - data: { - 'gen_ai.usage.input_tokens': 100 - // gen_ai.usage.input_tokens.cached is NOT present - } - }; - const schema = { type: 'number', lte: 'gen_ai.usage.input_tokens', optional: true }; - const result = attributeMatches(span, 'gen_ai.usage.input_tokens.cached', schema); - - if (result === true) { - console.log('✓ Test 18: Optional attribute missing - passes'); - passed++; - } else { - console.log('✗ Test 18: FAILED - Should pass when optional attribute is missing'); - failed++; - } -} - -// Test 19: Optional attribute present and valid - should pass -{ - const span = { - data: { - 'gen_ai.usage.input_tokens': 100, - 'gen_ai.usage.input_tokens.cached': 50 - } - }; - const schema = { type: 'number', lte: 'gen_ai.usage.input_tokens', optional: true }; - const result = attributeMatches(span, 'gen_ai.usage.input_tokens.cached', schema); - - if (result === true) { - console.log('✓ Test 19: Optional attribute present and valid (50 <= 100) - passes'); - passed++; - } else { - console.log('✗ Test 19: FAILED - Should pass when optional attribute is present and valid'); - failed++; - } -} - -// Test 20: Optional attribute present but invalid - should fail -{ - const span = { - data: { - 'gen_ai.usage.input_tokens': 50, - 'gen_ai.usage.input_tokens.cached': 100 // Invalid: 100 > 50 - } - }; - const schema = { type: 'number', lte: 'gen_ai.usage.input_tokens', optional: true }; - const result = attributeMatches(span, 'gen_ai.usage.input_tokens.cached', schema); - - if (result === false) { - console.log('✓ Test 20: Optional attribute present but invalid (100 > 50) - fails'); - passed++; - } else { - console.log('✗ Test 20: FAILED - Should fail when optional attribute is present but invalid'); - failed++; - } -} - -// Test 21: Required (non-optional) attribute missing - should fail -{ - const span = { - data: { - 'gen_ai.usage.input_tokens': 100 - // gen_ai.usage.input_tokens.cached is NOT present - } - }; - const schema = { type: 'number', lte: 'gen_ai.usage.input_tokens' }; // No optional: true - const result = attributeMatches(span, 'gen_ai.usage.input_tokens.cached', schema); - - if (result === false) { - console.log('✓ Test 21: Required (non-optional) attribute missing - fails'); - passed++; - } else { - console.log('✗ Test 21: FAILED - Should fail when required attribute is missing'); - failed++; - } -} - -// Test 22: Optional with json_array type - missing attribute -{ - const span = { - data: { - // gen_ai.request.messages is NOT present - } - }; - const schema = { type: 'json_array', min_length: 1, optional: true }; - const result = attributeMatches(span, 'gen_ai.request.messages', schema); - - if (result === true) { - console.log('✓ Test 22: Optional json_array attribute missing - passes'); - passed++; - } else { - console.log('✗ Test 22: FAILED - Should pass when optional json_array attribute is missing'); - failed++; - } -} - -// Test 23: Optional with plain_string type - missing attribute -{ - const span = { - data: { - // gen_ai.response.text is NOT present - } - }; - const schema = { type: 'plain_string', min_length: 1, optional: true }; - const result = attributeMatches(span, 'gen_ai.response.text', schema); - - if (result === true) { - console.log('✓ Test 23: Optional plain_string attribute missing - passes'); - passed++; - } else { - console.log('✗ Test 23: FAILED - Should pass when optional plain_string attribute is missing'); - failed++; - } -} - -// ============================================================================ -// JSON Array length_lte Constraint Tests -// ============================================================================ - -console.log('\n--- JSON Array length_lte Constraint Tests ---\n'); - -// Test 24: Valid length_lte constraint (array length 2 <= original_length 5) -{ - const span = { - data: { - 'gen_ai.request.messages': JSON.stringify([ - { role: 'system', content: 'Hello' }, - { role: 'user', content: 'Hi' } - ]), - 'gen_ai.request.messages.original_length': 5 - } - }; - const schema = { type: 'json_array', length_lte: 'gen_ai.request.messages.original_length' }; - const result = attributeMatches(span, 'gen_ai.request.messages', schema); - - if (result === true) { - console.log('✓ Test 24: Valid length_lte constraint (array length 2 <= original_length 5)'); - passed++; - } else { - console.log('✗ Test 24: FAILED - Should pass when array length <= other attribute'); - failed++; - } -} - -// Test 25: Invalid length_lte constraint (array length 3 > original_length 2) -{ - const span = { - data: { - 'gen_ai.request.messages': JSON.stringify([ - { role: 'system', content: 'Hello' }, - { role: 'user', content: 'Hi' }, - { role: 'assistant', content: 'Bye' } - ]), - 'gen_ai.request.messages.original_length': 2 - } - }; - const schema = { type: 'json_array', length_lte: 'gen_ai.request.messages.original_length' }; - const result = attributeMatches(span, 'gen_ai.request.messages', schema); - - if (result === false) { - console.log('✓ Test 25: Invalid length_lte constraint detected (array length 3 > original_length 2)'); - passed++; - } else { - console.log('✗ Test 25: FAILED - Should fail when array length > other attribute'); - failed++; - } -} - -// Test 26: Edge case - length_lte with equal values (array length 3 <= original_length 3) -{ - const span = { - data: { - 'gen_ai.request.messages': JSON.stringify([ - { role: 'system', content: 'Hello' }, - { role: 'user', content: 'Hi' }, - { role: 'assistant', content: 'Bye' } - ]), - 'gen_ai.request.messages.original_length': 3 - } - }; - const schema = { type: 'json_array', length_lte: 'gen_ai.request.messages.original_length' }; - const result = attributeMatches(span, 'gen_ai.request.messages', schema); - - if (result === true) { - console.log('✓ Test 26: Edge case - length_lte with equal values (array length 3 <= original_length 3)'); - passed++; - } else { - console.log('✗ Test 26: FAILED - Should pass when array length == other attribute'); - failed++; - } -} - -// Test 27: length_lte passes when other attribute is missing -{ - const span = { - data: { - 'gen_ai.request.messages': JSON.stringify([ - { role: 'user', content: 'Hi' } - ]) - // gen_ai.request.messages.original_length is NOT present - } - }; - const schema = { type: 'json_array', length_lte: 'gen_ai.request.messages.original_length' }; - const result = attributeMatches(span, 'gen_ai.request.messages', schema); - - if (result === true) { - console.log('✓ Test 27: length_lte passes when other attribute is missing'); - passed++; - } else { - console.log('✗ Test 27: FAILED - Should pass when other attribute is missing'); - failed++; - } -} - -// Test 28: length_lte combined with items_have -{ - const span = { - data: { - 'gen_ai.request.messages': JSON.stringify([ - { role: 'system', content: 'Hello' }, - { role: 'user', content: 'Hi' } - ]), - 'gen_ai.request.messages.original_length': 5 - } - }; - const schema = { - type: 'json_array', - length_lte: 'gen_ai.request.messages.original_length', - items_have: ['role', 'content'] - }; - const result = attributeMatches(span, 'gen_ai.request.messages', schema); - - if (result === true) { - console.log('✓ Test 28: length_lte combined with items_have - passes'); - passed++; - } else { - console.log('✗ Test 28: FAILED - Should pass with valid length_lte and items_have'); - failed++; - } -} - -// Test 29: length_lte passes but items_have fails -{ - const span = { - data: { - 'gen_ai.request.messages': JSON.stringify([ - { role: 'system' }, // Missing 'content' - { role: 'user', content: 'Hi' } - ]), - 'gen_ai.request.messages.original_length': 5 - } - }; - const schema = { - type: 'json_array', - length_lte: 'gen_ai.request.messages.original_length', - items_have: ['role', 'content'] - }; - const result = attributeMatches(span, 'gen_ai.request.messages', schema); - - if (result === false) { - console.log('✓ Test 29: length_lte passes but items_have fails - overall fails'); - passed++; - } else { - console.log('✗ Test 29: FAILED - Should fail when items_have validation fails'); - failed++; - } -} - -// ============================================================================ -// JSON Array contains Constraint Tests -// ============================================================================ - -console.log('\n--- JSON Array contains Constraint Tests ---\n'); - -// Test 30: Valid contains - JSON string contains "[Blob substitute]" -{ - const span = { - data: { - 'gen_ai.request.messages': JSON.stringify([ - { role: 'user', content: [{ type: 'text', text: 'Describe this image' }, { type: 'image', data: '[Blob substitute]' }] } - ]) - } - }; - const schema = { type: 'json_array', contains: '[Blob substitute]' }; - const result = attributeMatches(span, 'gen_ai.request.messages', schema); - - if (result === true) { - console.log('✓ Test 30: Valid contains - JSON string contains "[Blob substitute]"'); - passed++; - } else { - console.log('✗ Test 30: FAILED - Should pass when JSON contains the substring'); - failed++; - } -} - -// Test 31: Invalid contains - JSON string does not contain "[Blob substitute]" -{ - const span = { - data: { - 'gen_ai.request.messages': JSON.stringify([ - { role: 'user', content: 'Hello world' } - ]) - } - }; - const schema = { type: 'json_array', contains: '[Blob substitute]' }; - const result = attributeMatches(span, 'gen_ai.request.messages', schema); - - if (result === false) { - console.log('✓ Test 31: Invalid contains - JSON string does not contain "[Blob substitute]"'); - passed++; - } else { - console.log('✗ Test 31: FAILED - Should fail when JSON does not contain the substring'); - failed++; - } -} - -// Test 32: contains combined with min_length -{ - const span = { - data: { - 'gen_ai.request.messages': JSON.stringify([ - { role: 'system', content: 'You are helpful' }, - { role: 'user', content: '[Blob substitute]' } - ]) - } - }; - const schema = { type: 'json_array', min_length: 2, contains: '[Blob substitute]' }; - const result = attributeMatches(span, 'gen_ai.request.messages', schema); - - if (result === true) { - console.log('✓ Test 32: contains combined with min_length - passes'); - passed++; - } else { - console.log('✗ Test 32: FAILED - Should pass with valid contains and min_length'); - failed++; - } -} - -// Test 33: contains passes but min_length fails -{ - const span = { - data: { - 'gen_ai.request.messages': JSON.stringify([ - { role: 'user', content: '[Blob substitute]' } - ]) - } - }; - const schema = { type: 'json_array', min_length: 3, contains: '[Blob substitute]' }; - const result = attributeMatches(span, 'gen_ai.request.messages', schema); - - if (result === false) { - console.log('✓ Test 33: contains passes but min_length fails - overall fails'); - passed++; - } else { - console.log('✗ Test 33: FAILED - Should fail when min_length validation fails'); - failed++; - } -} - -console.log(`\n${passed} passed, ${failed} failed`); -process.exit(failed > 0 ? 1 : 0); diff --git a/sdks/js/anthropic/cases/1-simple.js b/sdks/js/anthropic/cases/1-simple.js deleted file mode 100644 index 0c8f009..0000000 --- a/sdks/js/anthropic/cases/1-simple.js +++ /dev/null @@ -1,42 +0,0 @@ -/** - * 1-simple: Basic Completion - * - * Tests a simple chat completion request with Anthropic AI SDK - * and verifies that Sentry captures the appropriate spans and AI monitoring data. - */ - -const { Sentry } = require("../setup"); -const Anthropic = require("@anthropic-ai/sdk"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, system, prompt } = inputs; - - // Initialize Anthropic client - const anthropic = new Anthropic({ - apiKey: process.env.ANTHROPIC_API_KEY, - }); - - // Make the API call - const message = await anthropic.messages.create({ - model: model, - max_tokens: 1024, - system: system, - messages: [ - { - role: "user", - content: prompt, - }, - ], - }); - - if (!message.content || message.content.length === 0) { - throw new Error("No completion returned from Anthropic"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${JSON.stringify(message.content[0])}`); - } -} - -module.exports = runTestCase("1-simple", testLogic, Sentry); diff --git a/sdks/js/anthropic/cases/10-binary-content-redaction.js b/sdks/js/anthropic/cases/10-binary-content-redaction.js deleted file mode 100644 index 1b147af..0000000 --- a/sdks/js/anthropic/cases/10-binary-content-redaction.js +++ /dev/null @@ -1,75 +0,0 @@ -/** - * 10-binary-content-redaction: Binary Content Redaction Test - * - * Tests that when binary data (such as images) is sent to an LLM via Anthropic, - * Sentry correctly redacts the binary content in the captured span data and - * replaces it with a substitute marker ("[Blob substitute]"). - */ - -const { Sentry } = require("../setup"); -const Anthropic = require("@anthropic-ai/sdk"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -/** - * Creates a minimal valid PNG image (10x10 red square). - * Returns base64-encoded PNG data. - */ -function createMinimalPng() { - // Minimal 10x10 red PNG image (base64) - // This is a pre-generated minimal valid PNG to avoid needing canvas/sharp dependencies - const minimalPngBase64 = - "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8z8BQz0AEYBxVSF+FABJADveWkH6oAAAAAElFTkSuQmCC"; - - return minimalPngBase64; -} - -async function testLogic(inputs) { - const { model, image_type } = inputs; - - // Initialize Anthropic client - const anthropic = new Anthropic({ - apiKey: process.env.ANTHROPIC_API_KEY, - }); - - // Create binary image data - const base64Image = createMinimalPng(); - - // Anthropic uses a specific content block format for images - // See: https://docs.anthropic.com/en/docs/build-with-claude/vision - const message = await anthropic.messages.create({ - model: model, - max_tokens: 1024, - messages: [ - { - role: "user", - content: [ - { - type: "image", - source: { - type: "base64", - media_type: `image/${image_type}`, - data: base64Image, - }, - }, - { - type: "text", - text: "What color is this image? Answer in one word.", - }, - ], - }, - ], - }); - - if (!message.content || message.content.length === 0) { - throw new Error("No completion returned from Anthropic"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${JSON.stringify(message.content[0])}`); - console.log( - ` Sent image with ${base64Image.length} characters of base64 data` - ); - } -} - -module.exports = runTestCase("10-binary-content-redaction", testLogic, Sentry); diff --git a/sdks/js/anthropic/cases/2-multi-step.js b/sdks/js/anthropic/cases/2-multi-step.js deleted file mode 100644 index 39b2951..0000000 --- a/sdks/js/anthropic/cases/2-multi-step.js +++ /dev/null @@ -1,75 +0,0 @@ -/** - * 2-multi-step: Multi-step Conversation - * - * Tests a multi-step conversation with conversation history using Anthropic SDK - * and verifies that Sentry captures all spans for both API calls. - */ - -const { Sentry } = require("../setup"); -const Anthropic = require("@anthropic-ai/sdk"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, system, first_prompt, second_prompt } = inputs; - - // Initialize Anthropic client - const anthropic = new Anthropic({ - apiKey: process.env.ANTHROPIC_API_KEY, - }); - - // First call - const firstMessage = await anthropic.messages.create({ - model: model, - max_tokens: 1024, - system: system, - messages: [ - { - role: "user", - content: first_prompt, - }, - ], - }); - - if (!firstMessage.content || firstMessage.content.length === 0) { - throw new Error("No completion returned from Anthropic (first call)"); - } - - const firstText = firstMessage.content[0].text; - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` First response: ${firstText}`); - } - - // Second call with conversation history - const secondMessage = await anthropic.messages.create({ - model: model, - max_tokens: 1024, - system: system, - messages: [ - { - role: "user", - content: first_prompt, - }, - { - role: "assistant", - content: firstText, - }, - { - role: "user", - content: second_prompt, - }, - ], - }); - - if (!secondMessage.content || secondMessage.content.length === 0) { - throw new Error("No completion returned from Anthropic (second call)"); - } - - const secondText = secondMessage.content[0].text; - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Second response: ${secondText}`); - } -} - -module.exports = runTestCase("2-multi-step", testLogic, Sentry); diff --git a/sdks/js/anthropic/cases/9-message-truncation.js b/sdks/js/anthropic/cases/9-message-truncation.js deleted file mode 100644 index 7b50440..0000000 --- a/sdks/js/anthropic/cases/9-message-truncation.js +++ /dev/null @@ -1,74 +0,0 @@ -/** - * 9-message-truncation: Message Truncation - * - * Tests that large messages are properly truncated while preserving the original message count. - * Sends multiple large messages (~9KB each) and verifies that Sentry captures: - * - gen_ai.request.messages.original_length (the actual count of messages sent) - * - gen_ai.request.messages array (potentially truncated for telemetry) - * - The relationship: len(messages) <= original_length - */ - -const { Sentry } = require("../setup"); -const Anthropic = require("@anthropic-ai/sdk"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, message_size_kb, message_count } = inputs; - - // Initialize Anthropic client - const anthropic = new Anthropic({ - apiKey: process.env.ANTHROPIC_API_KEY, - }); - - // Generate large message content (~9KB each) - // Using approximately 1 character per byte for ASCII - const contentSize = message_size_kb * 1024; - const largeContent = "A".repeat(contentSize); - - // Build messages array with alternating user/assistant messages - // Anthropic requires conversation to alternate between user and assistant - const messages = []; - for (let i = 0; i < message_count; i++) { - if (i % 2 === 0) { - // User messages - messages.push({ - role: "user", - content: `Message ${i + 1}: ${largeContent}`, - }); - } else { - // Assistant messages (for conversation history) - messages.push({ - role: "assistant", - content: `Message ${i + 1}: ${largeContent}`, - }); - } - } - - // Ensure the last message is from user (required by Anthropic) - if (messages[messages.length - 1].role === "assistant") { - messages.push({ - role: "user", - content: "Please summarize what we discussed.", - }); - } - - // Make the API call with large messages - const message = await anthropic.messages.create({ - model: model, - max_tokens: 1024, - messages: messages, - }); - - if (!message.content || message.content.length === 0) { - throw new Error("No completion returned from Anthropic"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${JSON.stringify(message.content[0])}`); - console.log( - ` Sent ${messages.length} messages with ~${message_size_kb}KB each` - ); - } -} - -module.exports = runTestCase("9-message-truncation", testLogic, Sentry); diff --git a/sdks/js/anthropic/config.json b/sdks/js/anthropic/config.json deleted file mode 100644 index a97f958..0000000 --- a/sdks/js/anthropic/config.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "sdk_name": "anthropic", - "framework_type": "low-level", - "metadata": { - "description": "Anthropic AI SDK for JavaScript - Low-level API client", - "notes": "Direct Claude API calls without agent wrappers" - }, - "overrides": { - "1-simple": { - "model": "claude-haiku-4-5", - "gen_ai.request.model": "claude-haiku-4-5", - "gen_ai.response.model": "claude-haiku-4-5*" - }, - "2-multi-step": { - "model": "claude-haiku-4-5", - "gen_ai.request.model": "claude-haiku-4-5", - "gen_ai.response.model": "claude-haiku-4-5*" - }, - "9-message-truncation": { - "model": "claude-haiku-4-5", - "gen_ai.request.model": "claude-haiku-4-5", - "gen_ai.response.model": "claude-haiku-4-5*" - }, - "10-binary-content-redaction": { - "model": "claude-haiku-4-5", - "gen_ai.request.model": "claude-haiku-4-5", - "gen_ai.response.model": "claude-haiku-4-5*" - } - } -} diff --git a/sdks/js/anthropic/package-lock.json b/sdks/js/anthropic/package-lock.json deleted file mode 100644 index 07a6959..0000000 --- a/sdks/js/anthropic/package-lock.json +++ /dev/null @@ -1,984 +0,0 @@ -{ - "name": "@sentry-ai-sdks/anthropic", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@sentry-ai-sdks/anthropic", - "version": "1.0.0", - "dependencies": { - "@anthropic-ai/sdk": "0.68.0", - "@sentry/node": "10.34.0", - "dotenv": "17.2.3" - } - }, - "node_modules/@anthropic-ai/sdk": { - "version": "0.68.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.68.0.tgz", - "integrity": "sha512-SMYAmbbiprG8k1EjEPMTwaTqssDT7Ae+jxcR5kWXiqTlbwMR2AthXtscEVWOHkRfyAV5+y3PFYTJRNa3OJWIEw==", - "license": "MIT", - "dependencies": { - "json-schema-to-ts": "^3.1.1" - }, - "bin": { - "anthropic-ai-sdk": "bin/cli" - }, - "peerDependencies": { - "zod": "^3.25.0 || ^4.0.0" - }, - "peerDependenciesMeta": { - "zod": { - "optional": true - } - } - }, - "node_modules/@apm-js-collab/code-transformer": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/@apm-js-collab/code-transformer/-/code-transformer-0.8.2.tgz", - "integrity": "sha512-YRjJjNq5KFSjDUoqu5pFUWrrsvGOxl6c3bu+uMFc9HNNptZ2rNU/TI2nLw4jnhQNtka972Ee2m3uqbvDQtPeCA==", - "license": "Apache-2.0" - }, - "node_modules/@apm-js-collab/tracing-hooks": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@apm-js-collab/tracing-hooks/-/tracing-hooks-0.3.1.tgz", - "integrity": "sha512-Vu1CbmPURlN5fTboVuKMoJjbO5qcq9fA5YXpskx3dXe/zTBvjODFoerw+69rVBlRLrJpwPqSDqEuJDEKIrTldw==", - "license": "Apache-2.0", - "dependencies": { - "@apm-js-collab/code-transformer": "^0.8.0", - "debug": "^4.4.1", - "module-details-from-path": "^1.0.4" - } - }, - "node_modules/@babel/runtime": { - "version": "7.28.4", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.4.tgz", - "integrity": "sha512-Q/N6JNWvIvPnLDvjlE1OUBLPQHH6l3CltCEsHIujp45zQUSSh8K+gHnaEX45yAT1nyngnINhvWtzN+Nb9D8RAQ==", - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@opentelemetry/api": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", - "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", - "license": "Apache-2.0", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/api-logs": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.208.0.tgz", - "integrity": "sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api": "^1.3.0" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/context-async-hooks": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.4.0.tgz", - "integrity": "sha512-jn0phJ+hU7ZuvaoZE/8/Euw3gvHJrn2yi+kXrymwObEPVPjtwCmkvXDRQCWli+fCTTF/aSOtXaLr7CLIvv3LQg==", - "license": "Apache-2.0", - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/core": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.2.0.tgz", - "integrity": "sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/instrumentation": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.208.0.tgz", - "integrity": "sha512-Eju0L4qWcQS+oXxi6pgh7zvE2byogAkcsVv0OjHF/97iOz1N/aKE6etSGowYkie+YA1uo6DNwdSxaaNnLvcRlA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api-logs": "0.208.0", - "import-in-the-middle": "^2.0.0", - "require-in-the-middle": "^8.0.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-amqplib": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-amqplib/-/instrumentation-amqplib-0.55.0.tgz", - "integrity": "sha512-5ULoU8p+tWcQw5PDYZn8rySptGSLZHNX/7srqo2TioPnAAcvTy6sQFQXsNPrAnyRRtYGMetXVyZUy5OaX1+IfA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-connect": { - "version": "0.52.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-connect/-/instrumentation-connect-0.52.0.tgz", - "integrity": "sha512-GXPxfNB5szMbV3I9b7kNWSmQBoBzw7MT0ui6iU/p+NIzVx3a06Ri2cdQO7tG9EKb4aKSLmfX9Cw5cKxXqX6Ohg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0", - "@types/connect": "3.4.38" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-dataloader": { - "version": "0.26.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-dataloader/-/instrumentation-dataloader-0.26.0.tgz", - "integrity": "sha512-P2BgnFfTOarZ5OKPmYfbXfDFjQ4P9WkQ1Jji7yH5/WwB6Wm/knynAoA1rxbjWcDlYupFkyT0M1j6XLzDzy0aCA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-express": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-express/-/instrumentation-express-0.57.0.tgz", - "integrity": "sha512-HAdx/o58+8tSR5iW+ru4PHnEejyKrAy9fYFhlEI81o10nYxrGahnMAHWiSjhDC7UQSY3I4gjcPgSKQz4rm/asg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-fs": { - "version": "0.28.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.28.0.tgz", - "integrity": "sha512-FFvg8fq53RRXVBRHZViP+EMxMR03tqzEGpuq55lHNbVPyFklSVfQBN50syPhK5UYYwaStx0eyCtHtbRreusc5g==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-generic-pool": { - "version": "0.52.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-generic-pool/-/instrumentation-generic-pool-0.52.0.tgz", - "integrity": "sha512-ISkNcv5CM2IwvsMVL31Tl61/p2Zm2I2NAsYq5SSBgOsOndT0TjnptjufYVScCnD5ZLD1tpl4T3GEYULLYOdIdQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-graphql": { - "version": "0.56.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-graphql/-/instrumentation-graphql-0.56.0.tgz", - "integrity": "sha512-IPvNk8AFoVzTAM0Z399t34VDmGDgwT6rIqCUug8P9oAGerl2/PEIYMPOl/rerPGu+q8gSWdmbFSjgg7PDVRd3Q==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-hapi": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-hapi/-/instrumentation-hapi-0.55.0.tgz", - "integrity": "sha512-prqAkRf9e4eEpy4G3UcR32prKE8NLNlA90TdEU1UsghOTg0jUvs40Jz8LQWFEs5NbLbXHYGzB4CYVkCI8eWEVQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-http": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.208.0.tgz", - "integrity": "sha512-rhmK46DRWEbQQB77RxmVXGyjs6783crXCnFjYQj+4tDH/Kpv9Rbg3h2kaNyp5Vz2emF1f9HOQQvZoHzwMWOFZQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/instrumentation": "0.208.0", - "@opentelemetry/semantic-conventions": "^1.29.0", - "forwarded-parse": "2.1.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-ioredis": { - "version": "0.56.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.56.0.tgz", - "integrity": "sha512-XSWeqsd3rKSsT3WBz/JKJDcZD4QYElZEa0xVdX8f9dh4h4QgXhKRLorVsVkK3uXFbC2sZKAS2Ds+YolGwD83Dg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/redis-common": "^0.38.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-kafkajs": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-kafkajs/-/instrumentation-kafkajs-0.18.0.tgz", - "integrity": "sha512-KCL/1HnZN5zkUMgPyOxfGjLjbXjpd4odDToy+7c+UsthIzVLFf99LnfIBE8YSSrYE4+uS7OwJMhvhg3tWjqMBg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.30.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-knex": { - "version": "0.53.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-knex/-/instrumentation-knex-0.53.0.tgz", - "integrity": "sha512-xngn5cH2mVXFmiT1XfQ1aHqq1m4xb5wvU6j9lSgLlihJ1bXzsO543cpDwjrZm2nMrlpddBf55w8+bfS4qDh60g==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.33.1" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-koa": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-koa/-/instrumentation-koa-0.57.0.tgz", - "integrity": "sha512-3JS8PU/D5E3q295mwloU2v7c7/m+DyCqdu62BIzWt+3u9utjxC9QS7v6WmUNuoDN3RM+Q+D1Gpj13ERo+m7CGg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.36.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0" - } - }, - "node_modules/@opentelemetry/instrumentation-lru-memoizer": { - "version": "0.53.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.53.0.tgz", - "integrity": "sha512-LDwWz5cPkWWr0HBIuZUjslyvijljTwmwiItpMTHujaULZCxcYE9eU44Qf/pbVC8TulT0IhZi+RoGvHKXvNhysw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mongodb": { - "version": "0.61.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.61.0.tgz", - "integrity": "sha512-OV3i2DSoY5M/pmLk+68xr5RvkHU8DRB3DKMzYJdwDdcxeLs62tLbkmRyqJZsYf3Ht7j11rq35pHOWLuLzXL7pQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mongoose": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.55.0.tgz", - "integrity": "sha512-5afj0HfF6aM6Nlqgu6/PPHFk8QBfIe3+zF9FGpX76jWPS0/dujoEYn82/XcLSaW5LPUDW8sni+YeK0vTBNri+w==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mysql": { - "version": "0.54.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql/-/instrumentation-mysql-0.54.0.tgz", - "integrity": "sha512-bqC1YhnwAeWmRzy1/Xf9cDqxNG2d/JDkaxnqF5N6iJKN1eVWI+vg7NfDkf52/Nggp3tl1jcC++ptC61BD6738A==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@types/mysql": "2.15.27" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mysql2": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql2/-/instrumentation-mysql2-0.55.0.tgz", - "integrity": "sha512-0cs8whQG55aIi20gnK8B7cco6OK6N+enNhW0p5284MvqJ5EPi+I1YlWsWXgzv/V2HFirEejkvKiI4Iw21OqDWg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.33.0", - "@opentelemetry/sql-common": "^0.41.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-pg": { - "version": "0.61.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.61.0.tgz", - "integrity": "sha512-UeV7KeTnRSM7ECHa3YscoklhUtTQPs6V6qYpG283AB7xpnPGCUCUfECFT9jFg6/iZOQTt3FHkB1wGTJCNZEvPw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.34.0", - "@opentelemetry/sql-common": "^0.41.2", - "@types/pg": "8.15.6", - "@types/pg-pool": "2.0.6" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-redis": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-redis/-/instrumentation-redis-0.57.0.tgz", - "integrity": "sha512-bCxTHQFXzrU3eU1LZnOZQ3s5LURxQPDlU3/upBzlWY77qOI1GZuGofazj3jtzjctMJeBEJhNwIFEgRPBX1kp/Q==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/redis-common": "^0.38.2", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-tedious": { - "version": "0.27.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-tedious/-/instrumentation-tedious-0.27.0.tgz", - "integrity": "sha512-jRtyUJNZppPBjPae4ZjIQ2eqJbcRaRfJkr0lQLHFmOU/no5A6e9s1OHLd5XZyZoBJ/ymngZitanyRRA5cniseA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@types/tedious": "^4.0.14" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-undici": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.19.0.tgz", - "integrity": "sha512-Pst/RhR61A2OoZQZkn6OLpdVpXp6qn3Y92wXa6umfJe9rV640r4bc6SWvw4pPN6DiQqPu2c8gnSSZPDtC6JlpQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.24.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.7.0" - } - }, - "node_modules/@opentelemetry/redis-common": { - "version": "0.38.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/redis-common/-/redis-common-0.38.2.tgz", - "integrity": "sha512-1BCcU93iwSRZvDAgwUxC/DV4T/406SkMfxGqu5ojc3AvNI+I9GhV7v0J1HljsczuuhcnFLYqD5VmwVXfCGHzxA==", - "license": "Apache-2.0", - "engines": { - "node": "^18.19.0 || >=20.6.0" - } - }, - "node_modules/@opentelemetry/resources": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.4.0.tgz", - "integrity": "sha512-RWvGLj2lMDZd7M/5tjkI/2VHMpXebLgPKvBUd9LRasEWR2xAynDwEYZuLvY9P2NGG73HF07jbbgWX2C9oavcQg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.4.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/resources/node_modules/@opentelemetry/core": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.4.0.tgz", - "integrity": "sha512-KtcyFHssTn5ZgDu6SXmUznS80OFs/wN7y6MyFRRcKU6TOw8hNcGxKvt8hsdaLJfhzUszNSjURetq5Qpkad14Gw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/sdk-trace-base": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.4.0.tgz", - "integrity": "sha512-WH0xXkz/OHORDLKqaxcUZS0X+t1s7gGlumr2ebiEgNZQl2b0upK2cdoD0tatf7l8iP74woGJ/Kmxe82jdvcWRw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.4.0", - "@opentelemetry/resources": "2.4.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/sdk-trace-base/node_modules/@opentelemetry/core": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.4.0.tgz", - "integrity": "sha512-KtcyFHssTn5ZgDu6SXmUznS80OFs/wN7y6MyFRRcKU6TOw8hNcGxKvt8hsdaLJfhzUszNSjURetq5Qpkad14Gw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/semantic-conventions": { - "version": "1.38.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.38.0.tgz", - "integrity": "sha512-kocjix+/sSggfJhwXqClZ3i9Y/MI0fp7b+g7kCRm6psy2dsf8uApTRclwG18h8Avm7C9+fnt+O36PspJ/OzoWg==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/@opentelemetry/sql-common": { - "version": "0.41.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/sql-common/-/sql-common-0.41.2.tgz", - "integrity": "sha512-4mhWm3Z8z+i508zQJ7r6Xi7y4mmoJpdvH0fZPFRkWrdp5fq7hhZ2HhYokEOLkfqSMgPR4Z9EyB3DBkbKGOqZiQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.1.0" - } - }, - "node_modules/@prisma/instrumentation": { - "version": "6.19.0", - "resolved": "https://registry.npmjs.org/@prisma/instrumentation/-/instrumentation-6.19.0.tgz", - "integrity": "sha512-QcuYy25pkXM8BJ37wVFBO7Zh34nyRV1GOb2n3lPkkbRYfl4hWl3PTcImP41P0KrzVXfa/45p6eVCos27x3exIg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": ">=0.52.0 <1" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.8" - } - }, - "node_modules/@sentry/core": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/core/-/core-10.34.0.tgz", - "integrity": "sha512-4FFpYBMf0VFdPcsr4grDYDOR87mRu6oCfb51oQjU/Pndmty7UgYo0Bst3LEC/8v0SpytBtzXq+Wx/fkwulBesg==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@sentry/node": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/node/-/node-10.34.0.tgz", - "integrity": "sha512-bEOyH97HuVtWZYAZ5mp0NhYNc+n6QCfiKuLee2P75n2kt4cIPTGvLOSdUwwjllf795uOdKZJuM1IUN0W+YMcVg==", - "license": "MIT", - "dependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^2.2.0", - "@opentelemetry/core": "^2.2.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/instrumentation-amqplib": "0.55.0", - "@opentelemetry/instrumentation-connect": "0.52.0", - "@opentelemetry/instrumentation-dataloader": "0.26.0", - "@opentelemetry/instrumentation-express": "0.57.0", - "@opentelemetry/instrumentation-fs": "0.28.0", - "@opentelemetry/instrumentation-generic-pool": "0.52.0", - "@opentelemetry/instrumentation-graphql": "0.56.0", - "@opentelemetry/instrumentation-hapi": "0.55.0", - "@opentelemetry/instrumentation-http": "0.208.0", - "@opentelemetry/instrumentation-ioredis": "0.56.0", - "@opentelemetry/instrumentation-kafkajs": "0.18.0", - "@opentelemetry/instrumentation-knex": "0.53.0", - "@opentelemetry/instrumentation-koa": "0.57.0", - "@opentelemetry/instrumentation-lru-memoizer": "0.53.0", - "@opentelemetry/instrumentation-mongodb": "0.61.0", - "@opentelemetry/instrumentation-mongoose": "0.55.0", - "@opentelemetry/instrumentation-mysql": "0.54.0", - "@opentelemetry/instrumentation-mysql2": "0.55.0", - "@opentelemetry/instrumentation-pg": "0.61.0", - "@opentelemetry/instrumentation-redis": "0.57.0", - "@opentelemetry/instrumentation-tedious": "0.27.0", - "@opentelemetry/instrumentation-undici": "0.19.0", - "@opentelemetry/resources": "^2.2.0", - "@opentelemetry/sdk-trace-base": "^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0", - "@prisma/instrumentation": "6.19.0", - "@sentry/core": "10.34.0", - "@sentry/node-core": "10.34.0", - "@sentry/opentelemetry": "10.34.0", - "import-in-the-middle": "^2.0.1", - "minimatch": "^9.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@sentry/node-core": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/node-core/-/node-core-10.34.0.tgz", - "integrity": "sha512-FrGfC8GzD1cnZDO3zwQ4cjyoY1ZwNHvZbXSvXRYxpjhXidZhvaPurjgLRSB0xGaFgoemmOp1ufsx/w6fQOGA6Q==", - "license": "MIT", - "dependencies": { - "@apm-js-collab/tracing-hooks": "^0.3.1", - "@sentry/core": "10.34.0", - "@sentry/opentelemetry": "10.34.0", - "import-in-the-middle": "^2.0.1" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/core": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/instrumentation": ">=0.57.1 <1", - "@opentelemetry/resources": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0" - } - }, - "node_modules/@sentry/opentelemetry": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/opentelemetry/-/opentelemetry-10.34.0.tgz", - "integrity": "sha512-uKuULBOmdVu3bYdD8doMLqKgN0PP3WWtI7Shu11P9PVrhSNT4U9yM9Z6v1aFlQcbrgyg3LynZuXs8lyjt90UbA==", - "license": "MIT", - "dependencies": { - "@sentry/core": "10.34.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/core": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0" - } - }, - "node_modules/@types/connect": { - "version": "3.4.38", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", - "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/mysql": { - "version": "2.15.27", - "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.27.tgz", - "integrity": "sha512-YfWiV16IY0OeBfBCk8+hXKmdTKrKlwKN1MNKAPBu5JYxLwBEZl7QzeEpGnlZb3VMGJrrGmB84gXiH+ofs/TezA==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/node": { - "version": "24.10.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", - "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", - "license": "MIT", - "dependencies": { - "undici-types": "~7.16.0" - } - }, - "node_modules/@types/pg": { - "version": "8.15.6", - "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.15.6.tgz", - "integrity": "sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "pg-protocol": "*", - "pg-types": "^2.2.0" - } - }, - "node_modules/@types/pg-pool": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@types/pg-pool/-/pg-pool-2.0.6.tgz", - "integrity": "sha512-TaAUE5rq2VQYxab5Ts7WZhKNmuN78Q6PiFonTDdpbx8a1H0M1vhy3rhiMjl+e2iHmogyMw7jZF4FrE6eJUy5HQ==", - "license": "MIT", - "dependencies": { - "@types/pg": "*" - } - }, - "node_modules/@types/tedious": { - "version": "4.0.14", - "resolved": "https://registry.npmjs.org/@types/tedious/-/tedious-4.0.14.tgz", - "integrity": "sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", - "license": "MIT", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-import-attributes": { - "version": "1.9.5", - "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", - "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", - "license": "MIT", - "peerDependencies": { - "acorn": "^8" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "license": "MIT" - }, - "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/cjs-module-lexer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.2.0.tgz", - "integrity": "sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==", - "license": "MIT" - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/dotenv": { - "version": "17.2.3", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", - "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/forwarded-parse": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/forwarded-parse/-/forwarded-parse-2.1.2.tgz", - "integrity": "sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==", - "license": "MIT" - }, - "node_modules/import-in-the-middle": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-2.0.4.tgz", - "integrity": "sha512-Al0kMpa0BqfvDnxjxGlab9vdQ0vTDs82TBKrD59X9jReUoPAzSGBb6vGDzMUMFBGyyDF03RpLT4oxGn6BpASzQ==", - "license": "Apache-2.0", - "dependencies": { - "acorn": "^8.15.0", - "acorn-import-attributes": "^1.9.5", - "cjs-module-lexer": "^2.2.0", - "module-details-from-path": "^1.0.4" - } - }, - "node_modules/json-schema-to-ts": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", - "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", - "license": "MIT", - "dependencies": { - "@babel/runtime": "^7.18.3", - "ts-algebra": "^2.0.0" - }, - "engines": { - "node": ">=16" - } - }, - "node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/module-details-from-path": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", - "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", - "license": "MIT" - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/pg-int8": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", - "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", - "license": "ISC", - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/pg-protocol": { - "version": "1.10.3", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.10.3.tgz", - "integrity": "sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==", - "license": "MIT" - }, - "node_modules/pg-types": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", - "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", - "license": "MIT", - "dependencies": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-array": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", - "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-bytea": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", - "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-date": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", - "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-interval": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", - "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", - "license": "MIT", - "dependencies": { - "xtend": "^4.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-in-the-middle": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-8.0.1.tgz", - "integrity": "sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==", - "license": "MIT", - "dependencies": { - "debug": "^4.3.5", - "module-details-from-path": "^1.0.3" - }, - "engines": { - "node": ">=9.3.0 || >=8.10.0 <9.0.0" - } - }, - "node_modules/ts-algebra": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", - "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", - "license": "MIT" - }, - "node_modules/undici-types": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", - "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", - "license": "MIT" - }, - "node_modules/xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "license": "MIT", - "engines": { - "node": ">=0.4" - } - } - } -} diff --git a/sdks/js/anthropic/package.json b/sdks/js/anthropic/package.json deleted file mode 100644 index 365204f..0000000 --- a/sdks/js/anthropic/package.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "@sentry-ai-sdks/anthropic", - "version": "1.0.0", - "description": "Sentry integration tests for Anthropic AI SDK", - "scripts": { - "test": "echo \"Use the CLI: npm run cli run --sdk js/anthropic\"" - }, - "dependencies": { - "@anthropic-ai/sdk": "0.68.0", - "@sentry/node": "10.34.0", - "dotenv": "17.2.3" - } -} diff --git a/sdks/js/anthropic/setup.js b/sdks/js/anthropic/setup.js deleted file mode 100644 index 272e55a..0000000 --- a/sdks/js/anthropic/setup.js +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Setup file for Anthropic AI SDK tests - * - * Initializes Sentry with Anthropic AI-specific integrations. - */ - -const Sentry = require("@sentry/node"); -const { createTransport } = require("@sentry/core"); -const { config } = require("dotenv"); -const { resolve } = require("path"); -const { createMockTransport } = require("../_test-utils/mock-transport.cjs"); - -// Load environment variables -config({ quiet: true, path: resolve(__dirname, ".env") }); - -// Initialize Sentry. -// NOTE: AI integrations are auto-enabled in the Node.js SDK and -// therefore do not need to be configured explicitly. -Sentry.init({ - dsn: process.env.SENTRY_DSN || "https://public@127.0.0.1/1", - tracesSampleRate: 1.0, - transport: createMockTransport(createTransport), - sendDefaultPii: true, -}); - -module.exports = { Sentry }; diff --git a/sdks/js/google-genai/cases/1-simple.js b/sdks/js/google-genai/cases/1-simple.js deleted file mode 100644 index 76bf63c..0000000 --- a/sdks/js/google-genai/cases/1-simple.js +++ /dev/null @@ -1,38 +0,0 @@ -/** - * 1-simple: Basic Completion - * - * Tests a simple chat completion request with Google GenAI SDK - * and verifies that Sentry captures the appropriate spans and AI monitoring data. - */ - -const { Sentry } = require("../setup"); -const { GoogleGenAI } = require("@google/genai"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, prompt, system } = inputs; - - const client = new GoogleGenAI({ - apiKey: process.env.GOOGLE_GENAI_API_KEY, - }); - - const response = await client.models.generateContent({ - model, - contents: prompt, - config: { - systemInstruction: [ system ], - }, - }); - - const text = response.text; - - if (!text) { - throw new Error("No completion returned from Google GenAI"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${text}`); - } -} - -module.exports = runTestCase("1-simple", testLogic, Sentry); diff --git a/sdks/js/google-genai/cases/10-binary-content-redaction.js b/sdks/js/google-genai/cases/10-binary-content-redaction.js deleted file mode 100644 index ceb0503..0000000 --- a/sdks/js/google-genai/cases/10-binary-content-redaction.js +++ /dev/null @@ -1,59 +0,0 @@ -/** - * 10-binary-content-redaction: Binary Content Redaction - * - * Tests that binary image data is properly redacted in Sentry spans. - * Sends a message with binary image data and verifies that Sentry: - * - Captures the message structure - * - Redacts the binary content with "[Blob substitute]" marker - * - Does not send raw binary data to Sentry - */ - -const { Sentry } = require("../setup"); -const { GoogleGenAI } = require("@google/genai"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, image_type } = inputs; - - const client = new GoogleGenAI({ - apiKey: process.env.GOOGLE_GENAI_API_KEY, - }); - - // Create a small base64-encoded image (1x1 pixel PNG) - // This is a valid 1x1 transparent PNG image - const base64Image = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; - - // Convert base64 to binary data - const imageData = Buffer.from(base64Image, "base64"); - - // Google GenAI accepts inline data as part of the contents array - // Contents can be an array of Part objects with inlineData or text - const response = await client.models.generateContent({ - model, - contents: [ - { - inlineData: { - mimeType: `image/${image_type}`, - data: base64Image, - }, - }, - { - text: "What do you see in this image? Describe briefly.", - }, - ], - }); - - const text = response.text; - - if (!text) { - throw new Error("No completion returned from Google GenAI"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${text}`); - console.log(` Sent message with base64-encoded ${image_type} image (${imageData.length} bytes)`); - } -} - -module.exports = runTestCase("10-binary-content-redaction", testLogic, Sentry); diff --git a/sdks/js/google-genai/cases/2-multi-step.js b/sdks/js/google-genai/cases/2-multi-step.js deleted file mode 100644 index 9155d4b..0000000 --- a/sdks/js/google-genai/cases/2-multi-step.js +++ /dev/null @@ -1,62 +0,0 @@ -/** - * 2-multi-step: Multi-step Conversation - * - * Tests a multi-step conversation with conversation history using Google GenAI SDK - * and verifies that Sentry captures all spans for both API calls. - */ - -const { Sentry } = require("../setup"); -const { GoogleGenAI } = require("@google/genai"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, system, first_prompt, second_prompt } = inputs; - - const client = new GoogleGenAI({ - apiKey: process.env.GOOGLE_GENAI_API_KEY, - }); - - // First call - const firstResponse = await client.models.generateContent({ - model, - contents: first_prompt, - config: { - systemInstruction: [system], - }, - }); - - const firstText = firstResponse.text; - - if (!firstText) { - throw new Error("No completion returned from Google GenAI (first call)"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` First response: ${firstText}`); - } - - // Second call with conversation history - const secondResponse = await client.models.generateContent({ - model, - contents: [ - { role: "user", parts: [{ text: first_prompt }] }, - { role: "model", parts: [{ text: firstText }] }, - { role: "user", parts: [{ text: second_prompt }] }, - ], - config: { - systemInstruction: [system], - }, - }); - - const secondText = secondResponse.text; - - if (!secondText) { - throw new Error("No completion returned from Google GenAI (second call)"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Second response: ${secondText}`); - } -} - -module.exports = runTestCase("2-multi-step", testLogic, Sentry); diff --git a/sdks/js/google-genai/cases/9-message-truncation.js b/sdks/js/google-genai/cases/9-message-truncation.js deleted file mode 100644 index 206aeb1..0000000 --- a/sdks/js/google-genai/cases/9-message-truncation.js +++ /dev/null @@ -1,53 +0,0 @@ -/** - * 9-message-truncation: Message Truncation - * - * Tests that large messages are properly truncated while preserving the original message count. - * Sends multiple large messages (~9KB each) and verifies that Sentry captures: - * - gen_ai.request.messages.original_length (the actual count of messages sent) - * - gen_ai.request.messages array (potentially truncated for telemetry) - * - The relationship: len(messages) <= original_length - */ - -const { Sentry } = require("../setup"); -const { GoogleGenAI } = require("@google/genai"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, message_size_kb, message_count } = inputs; - - const client = new GoogleGenAI({ - apiKey: process.env.GOOGLE_GENAI_API_KEY, - }); - - // Generate large content for each message (~9KB each) - const contentSize = message_size_kb * 1024; - const largeContent = "x".repeat(contentSize); - - // Build a large prompt with multiple "messages" embedded - // Google GenAI treats contents as a single string or array of Content objects - // For testing message truncation, we'll embed multiple messages in the prompt - const promptParts = []; - for (let i = 0; i < message_count; i++) { - promptParts.push(`Message ${i + 1}: ${largeContent}`); - } - - const largePrompt = promptParts.join("\n\n") + "\n\nPlease summarize the above messages briefly."; - - const response = await client.models.generateContent({ - model, - contents: largePrompt, - }); - - const text = response.text; - - if (!text) { - throw new Error("No completion returned from Google GenAI"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${text.substring(0, 100)}...`); - console.log(` Sent ${message_count} messages with ~${message_size_kb}KB content each`); - } -} - -module.exports = runTestCase("9-message-truncation", testLogic, Sentry); diff --git a/sdks/js/google-genai/config.json b/sdks/js/google-genai/config.json deleted file mode 100644 index 8b8395e..0000000 --- a/sdks/js/google-genai/config.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "sdk_name": "google-genai", - "framework_type": "low-level", - "overrides": { - "1-simple": { - "model": "gemini-2.5-flash-lite", - "gen_ai.request.model": "gemini-2.5-flash-lite", - "gen_ai.response.model": "gemini-2.5-flash-lite" - }, - "2-multi-step": { - "model": "gemini-2.5-flash-lite", - "gen_ai.request.model": "gemini-2.5-flash-lite", - "gen_ai.response.model": "gemini-2.5-flash-lite" - }, - "9-message-truncation": { - "model": "gemini-2.5-flash-lite", - "gen_ai.request.model": "gemini-2.5-flash-lite", - "gen_ai.response.model": "gemini-2.5-flash-lite" - }, - "10-binary-content-redaction": { - "model": "gemini-2.5-flash-lite", - "gen_ai.request.model": "gemini-2.5-flash-lite", - "gen_ai.response.model": "gemini-2.5-flash-lite" - } - } -} diff --git a/sdks/js/google-genai/package-lock.json b/sdks/js/google-genai/package-lock.json deleted file mode 100644 index 0bb0aa5..0000000 --- a/sdks/js/google-genai/package-lock.json +++ /dev/null @@ -1,1717 +0,0 @@ -{ - "name": "@sentry-ai-sdks/google-genai", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@sentry-ai-sdks/google-genai", - "version": "1.0.0", - "dependencies": { - "@google/genai": "1.29.0", - "@sentry/node": "10.34.0", - "dotenv": "17.2.3" - } - }, - "node_modules/@apm-js-collab/code-transformer": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/@apm-js-collab/code-transformer/-/code-transformer-0.8.2.tgz", - "integrity": "sha512-YRjJjNq5KFSjDUoqu5pFUWrrsvGOxl6c3bu+uMFc9HNNptZ2rNU/TI2nLw4jnhQNtka972Ee2m3uqbvDQtPeCA==", - "license": "Apache-2.0" - }, - "node_modules/@apm-js-collab/tracing-hooks": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@apm-js-collab/tracing-hooks/-/tracing-hooks-0.3.1.tgz", - "integrity": "sha512-Vu1CbmPURlN5fTboVuKMoJjbO5qcq9fA5YXpskx3dXe/zTBvjODFoerw+69rVBlRLrJpwPqSDqEuJDEKIrTldw==", - "license": "Apache-2.0", - "dependencies": { - "@apm-js-collab/code-transformer": "^0.8.0", - "debug": "^4.4.1", - "module-details-from-path": "^1.0.4" - } - }, - "node_modules/@google/genai": { - "version": "1.29.0", - "resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.29.0.tgz", - "integrity": "sha512-cQP7Ssa06W+MSAyVtL/812FBtZDoDehnFObIpK1xo5Uv4XvqBcVZ8OhXgihOIXWn7xvPQGvLclR8+yt3Ysnd9g==", - "license": "Apache-2.0", - "dependencies": { - "google-auth-library": "^10.3.0", - "ws": "^8.18.0" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@modelcontextprotocol/sdk": "^1.20.1" - }, - "peerDependenciesMeta": { - "@modelcontextprotocol/sdk": { - "optional": true - } - } - }, - "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "license": "ISC", - "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@opentelemetry/api": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", - "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", - "license": "Apache-2.0", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/api-logs": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.208.0.tgz", - "integrity": "sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api": "^1.3.0" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/context-async-hooks": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.4.0.tgz", - "integrity": "sha512-jn0phJ+hU7ZuvaoZE/8/Euw3gvHJrn2yi+kXrymwObEPVPjtwCmkvXDRQCWli+fCTTF/aSOtXaLr7CLIvv3LQg==", - "license": "Apache-2.0", - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/core": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.2.0.tgz", - "integrity": "sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/instrumentation": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.208.0.tgz", - "integrity": "sha512-Eju0L4qWcQS+oXxi6pgh7zvE2byogAkcsVv0OjHF/97iOz1N/aKE6etSGowYkie+YA1uo6DNwdSxaaNnLvcRlA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api-logs": "0.208.0", - "import-in-the-middle": "^2.0.0", - "require-in-the-middle": "^8.0.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-amqplib": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-amqplib/-/instrumentation-amqplib-0.55.0.tgz", - "integrity": "sha512-5ULoU8p+tWcQw5PDYZn8rySptGSLZHNX/7srqo2TioPnAAcvTy6sQFQXsNPrAnyRRtYGMetXVyZUy5OaX1+IfA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-connect": { - "version": "0.52.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-connect/-/instrumentation-connect-0.52.0.tgz", - "integrity": "sha512-GXPxfNB5szMbV3I9b7kNWSmQBoBzw7MT0ui6iU/p+NIzVx3a06Ri2cdQO7tG9EKb4aKSLmfX9Cw5cKxXqX6Ohg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0", - "@types/connect": "3.4.38" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-dataloader": { - "version": "0.26.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-dataloader/-/instrumentation-dataloader-0.26.0.tgz", - "integrity": "sha512-P2BgnFfTOarZ5OKPmYfbXfDFjQ4P9WkQ1Jji7yH5/WwB6Wm/knynAoA1rxbjWcDlYupFkyT0M1j6XLzDzy0aCA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-express": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-express/-/instrumentation-express-0.57.0.tgz", - "integrity": "sha512-HAdx/o58+8tSR5iW+ru4PHnEejyKrAy9fYFhlEI81o10nYxrGahnMAHWiSjhDC7UQSY3I4gjcPgSKQz4rm/asg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-fs": { - "version": "0.28.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.28.0.tgz", - "integrity": "sha512-FFvg8fq53RRXVBRHZViP+EMxMR03tqzEGpuq55lHNbVPyFklSVfQBN50syPhK5UYYwaStx0eyCtHtbRreusc5g==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-generic-pool": { - "version": "0.52.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-generic-pool/-/instrumentation-generic-pool-0.52.0.tgz", - "integrity": "sha512-ISkNcv5CM2IwvsMVL31Tl61/p2Zm2I2NAsYq5SSBgOsOndT0TjnptjufYVScCnD5ZLD1tpl4T3GEYULLYOdIdQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-graphql": { - "version": "0.56.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-graphql/-/instrumentation-graphql-0.56.0.tgz", - "integrity": "sha512-IPvNk8AFoVzTAM0Z399t34VDmGDgwT6rIqCUug8P9oAGerl2/PEIYMPOl/rerPGu+q8gSWdmbFSjgg7PDVRd3Q==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-hapi": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-hapi/-/instrumentation-hapi-0.55.0.tgz", - "integrity": "sha512-prqAkRf9e4eEpy4G3UcR32prKE8NLNlA90TdEU1UsghOTg0jUvs40Jz8LQWFEs5NbLbXHYGzB4CYVkCI8eWEVQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-http": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.208.0.tgz", - "integrity": "sha512-rhmK46DRWEbQQB77RxmVXGyjs6783crXCnFjYQj+4tDH/Kpv9Rbg3h2kaNyp5Vz2emF1f9HOQQvZoHzwMWOFZQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/instrumentation": "0.208.0", - "@opentelemetry/semantic-conventions": "^1.29.0", - "forwarded-parse": "2.1.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-ioredis": { - "version": "0.56.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.56.0.tgz", - "integrity": "sha512-XSWeqsd3rKSsT3WBz/JKJDcZD4QYElZEa0xVdX8f9dh4h4QgXhKRLorVsVkK3uXFbC2sZKAS2Ds+YolGwD83Dg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/redis-common": "^0.38.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-kafkajs": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-kafkajs/-/instrumentation-kafkajs-0.18.0.tgz", - "integrity": "sha512-KCL/1HnZN5zkUMgPyOxfGjLjbXjpd4odDToy+7c+UsthIzVLFf99LnfIBE8YSSrYE4+uS7OwJMhvhg3tWjqMBg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.30.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-knex": { - "version": "0.53.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-knex/-/instrumentation-knex-0.53.0.tgz", - "integrity": "sha512-xngn5cH2mVXFmiT1XfQ1aHqq1m4xb5wvU6j9lSgLlihJ1bXzsO543cpDwjrZm2nMrlpddBf55w8+bfS4qDh60g==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.33.1" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-koa": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-koa/-/instrumentation-koa-0.57.0.tgz", - "integrity": "sha512-3JS8PU/D5E3q295mwloU2v7c7/m+DyCqdu62BIzWt+3u9utjxC9QS7v6WmUNuoDN3RM+Q+D1Gpj13ERo+m7CGg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.36.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0" - } - }, - "node_modules/@opentelemetry/instrumentation-lru-memoizer": { - "version": "0.53.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.53.0.tgz", - "integrity": "sha512-LDwWz5cPkWWr0HBIuZUjslyvijljTwmwiItpMTHujaULZCxcYE9eU44Qf/pbVC8TulT0IhZi+RoGvHKXvNhysw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mongodb": { - "version": "0.61.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.61.0.tgz", - "integrity": "sha512-OV3i2DSoY5M/pmLk+68xr5RvkHU8DRB3DKMzYJdwDdcxeLs62tLbkmRyqJZsYf3Ht7j11rq35pHOWLuLzXL7pQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mongoose": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.55.0.tgz", - "integrity": "sha512-5afj0HfF6aM6Nlqgu6/PPHFk8QBfIe3+zF9FGpX76jWPS0/dujoEYn82/XcLSaW5LPUDW8sni+YeK0vTBNri+w==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mysql": { - "version": "0.54.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql/-/instrumentation-mysql-0.54.0.tgz", - "integrity": "sha512-bqC1YhnwAeWmRzy1/Xf9cDqxNG2d/JDkaxnqF5N6iJKN1eVWI+vg7NfDkf52/Nggp3tl1jcC++ptC61BD6738A==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@types/mysql": "2.15.27" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mysql2": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql2/-/instrumentation-mysql2-0.55.0.tgz", - "integrity": "sha512-0cs8whQG55aIi20gnK8B7cco6OK6N+enNhW0p5284MvqJ5EPi+I1YlWsWXgzv/V2HFirEejkvKiI4Iw21OqDWg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.33.0", - "@opentelemetry/sql-common": "^0.41.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-pg": { - "version": "0.61.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.61.0.tgz", - "integrity": "sha512-UeV7KeTnRSM7ECHa3YscoklhUtTQPs6V6qYpG283AB7xpnPGCUCUfECFT9jFg6/iZOQTt3FHkB1wGTJCNZEvPw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.34.0", - "@opentelemetry/sql-common": "^0.41.2", - "@types/pg": "8.15.6", - "@types/pg-pool": "2.0.6" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-redis": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-redis/-/instrumentation-redis-0.57.0.tgz", - "integrity": "sha512-bCxTHQFXzrU3eU1LZnOZQ3s5LURxQPDlU3/upBzlWY77qOI1GZuGofazj3jtzjctMJeBEJhNwIFEgRPBX1kp/Q==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/redis-common": "^0.38.2", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-tedious": { - "version": "0.27.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-tedious/-/instrumentation-tedious-0.27.0.tgz", - "integrity": "sha512-jRtyUJNZppPBjPae4ZjIQ2eqJbcRaRfJkr0lQLHFmOU/no5A6e9s1OHLd5XZyZoBJ/ymngZitanyRRA5cniseA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@types/tedious": "^4.0.14" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-undici": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.19.0.tgz", - "integrity": "sha512-Pst/RhR61A2OoZQZkn6OLpdVpXp6qn3Y92wXa6umfJe9rV640r4bc6SWvw4pPN6DiQqPu2c8gnSSZPDtC6JlpQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.24.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.7.0" - } - }, - "node_modules/@opentelemetry/redis-common": { - "version": "0.38.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/redis-common/-/redis-common-0.38.2.tgz", - "integrity": "sha512-1BCcU93iwSRZvDAgwUxC/DV4T/406SkMfxGqu5ojc3AvNI+I9GhV7v0J1HljsczuuhcnFLYqD5VmwVXfCGHzxA==", - "license": "Apache-2.0", - "engines": { - "node": "^18.19.0 || >=20.6.0" - } - }, - "node_modules/@opentelemetry/resources": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.4.0.tgz", - "integrity": "sha512-RWvGLj2lMDZd7M/5tjkI/2VHMpXebLgPKvBUd9LRasEWR2xAynDwEYZuLvY9P2NGG73HF07jbbgWX2C9oavcQg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.4.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/resources/node_modules/@opentelemetry/core": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.4.0.tgz", - "integrity": "sha512-KtcyFHssTn5ZgDu6SXmUznS80OFs/wN7y6MyFRRcKU6TOw8hNcGxKvt8hsdaLJfhzUszNSjURetq5Qpkad14Gw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/sdk-trace-base": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.4.0.tgz", - "integrity": "sha512-WH0xXkz/OHORDLKqaxcUZS0X+t1s7gGlumr2ebiEgNZQl2b0upK2cdoD0tatf7l8iP74woGJ/Kmxe82jdvcWRw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.4.0", - "@opentelemetry/resources": "2.4.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/sdk-trace-base/node_modules/@opentelemetry/core": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.4.0.tgz", - "integrity": "sha512-KtcyFHssTn5ZgDu6SXmUznS80OFs/wN7y6MyFRRcKU6TOw8hNcGxKvt8hsdaLJfhzUszNSjURetq5Qpkad14Gw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/semantic-conventions": { - "version": "1.38.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.38.0.tgz", - "integrity": "sha512-kocjix+/sSggfJhwXqClZ3i9Y/MI0fp7b+g7kCRm6psy2dsf8uApTRclwG18h8Avm7C9+fnt+O36PspJ/OzoWg==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/@opentelemetry/sql-common": { - "version": "0.41.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/sql-common/-/sql-common-0.41.2.tgz", - "integrity": "sha512-4mhWm3Z8z+i508zQJ7r6Xi7y4mmoJpdvH0fZPFRkWrdp5fq7hhZ2HhYokEOLkfqSMgPR4Z9EyB3DBkbKGOqZiQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.1.0" - } - }, - "node_modules/@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">=14" - } - }, - "node_modules/@prisma/instrumentation": { - "version": "6.19.0", - "resolved": "https://registry.npmjs.org/@prisma/instrumentation/-/instrumentation-6.19.0.tgz", - "integrity": "sha512-QcuYy25pkXM8BJ37wVFBO7Zh34nyRV1GOb2n3lPkkbRYfl4hWl3PTcImP41P0KrzVXfa/45p6eVCos27x3exIg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": ">=0.52.0 <1" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.8" - } - }, - "node_modules/@sentry/core": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/core/-/core-10.34.0.tgz", - "integrity": "sha512-4FFpYBMf0VFdPcsr4grDYDOR87mRu6oCfb51oQjU/Pndmty7UgYo0Bst3LEC/8v0SpytBtzXq+Wx/fkwulBesg==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@sentry/node": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/node/-/node-10.34.0.tgz", - "integrity": "sha512-bEOyH97HuVtWZYAZ5mp0NhYNc+n6QCfiKuLee2P75n2kt4cIPTGvLOSdUwwjllf795uOdKZJuM1IUN0W+YMcVg==", - "license": "MIT", - "dependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^2.2.0", - "@opentelemetry/core": "^2.2.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/instrumentation-amqplib": "0.55.0", - "@opentelemetry/instrumentation-connect": "0.52.0", - "@opentelemetry/instrumentation-dataloader": "0.26.0", - "@opentelemetry/instrumentation-express": "0.57.0", - "@opentelemetry/instrumentation-fs": "0.28.0", - "@opentelemetry/instrumentation-generic-pool": "0.52.0", - "@opentelemetry/instrumentation-graphql": "0.56.0", - "@opentelemetry/instrumentation-hapi": "0.55.0", - "@opentelemetry/instrumentation-http": "0.208.0", - "@opentelemetry/instrumentation-ioredis": "0.56.0", - "@opentelemetry/instrumentation-kafkajs": "0.18.0", - "@opentelemetry/instrumentation-knex": "0.53.0", - "@opentelemetry/instrumentation-koa": "0.57.0", - "@opentelemetry/instrumentation-lru-memoizer": "0.53.0", - "@opentelemetry/instrumentation-mongodb": "0.61.0", - "@opentelemetry/instrumentation-mongoose": "0.55.0", - "@opentelemetry/instrumentation-mysql": "0.54.0", - "@opentelemetry/instrumentation-mysql2": "0.55.0", - "@opentelemetry/instrumentation-pg": "0.61.0", - "@opentelemetry/instrumentation-redis": "0.57.0", - "@opentelemetry/instrumentation-tedious": "0.27.0", - "@opentelemetry/instrumentation-undici": "0.19.0", - "@opentelemetry/resources": "^2.2.0", - "@opentelemetry/sdk-trace-base": "^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0", - "@prisma/instrumentation": "6.19.0", - "@sentry/core": "10.34.0", - "@sentry/node-core": "10.34.0", - "@sentry/opentelemetry": "10.34.0", - "import-in-the-middle": "^2.0.1", - "minimatch": "^9.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@sentry/node-core": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/node-core/-/node-core-10.34.0.tgz", - "integrity": "sha512-FrGfC8GzD1cnZDO3zwQ4cjyoY1ZwNHvZbXSvXRYxpjhXidZhvaPurjgLRSB0xGaFgoemmOp1ufsx/w6fQOGA6Q==", - "license": "MIT", - "dependencies": { - "@apm-js-collab/tracing-hooks": "^0.3.1", - "@sentry/core": "10.34.0", - "@sentry/opentelemetry": "10.34.0", - "import-in-the-middle": "^2.0.1" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/core": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/instrumentation": ">=0.57.1 <1", - "@opentelemetry/resources": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0" - } - }, - "node_modules/@sentry/opentelemetry": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/opentelemetry/-/opentelemetry-10.34.0.tgz", - "integrity": "sha512-uKuULBOmdVu3bYdD8doMLqKgN0PP3WWtI7Shu11P9PVrhSNT4U9yM9Z6v1aFlQcbrgyg3LynZuXs8lyjt90UbA==", - "license": "MIT", - "dependencies": { - "@sentry/core": "10.34.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/core": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0" - } - }, - "node_modules/@types/connect": { - "version": "3.4.38", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", - "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/mysql": { - "version": "2.15.27", - "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.27.tgz", - "integrity": "sha512-YfWiV16IY0OeBfBCk8+hXKmdTKrKlwKN1MNKAPBu5JYxLwBEZl7QzeEpGnlZb3VMGJrrGmB84gXiH+ofs/TezA==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/node": { - "version": "24.10.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", - "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", - "license": "MIT", - "dependencies": { - "undici-types": "~7.16.0" - } - }, - "node_modules/@types/pg": { - "version": "8.15.6", - "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.15.6.tgz", - "integrity": "sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "pg-protocol": "*", - "pg-types": "^2.2.0" - } - }, - "node_modules/@types/pg-pool": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@types/pg-pool/-/pg-pool-2.0.6.tgz", - "integrity": "sha512-TaAUE5rq2VQYxab5Ts7WZhKNmuN78Q6PiFonTDdpbx8a1H0M1vhy3rhiMjl+e2iHmogyMw7jZF4FrE6eJUy5HQ==", - "license": "MIT", - "dependencies": { - "@types/pg": "*" - } - }, - "node_modules/@types/tedious": { - "version": "4.0.14", - "resolved": "https://registry.npmjs.org/@types/tedious/-/tedious-4.0.14.tgz", - "integrity": "sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", - "license": "MIT", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-import-attributes": { - "version": "1.9.5", - "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", - "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", - "license": "MIT", - "peerDependencies": { - "acorn": "^8" - } - }, - "node_modules/agent-base": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", - "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/ansi-regex": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", - "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/ansi-styles": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "license": "MIT" - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/bignumber.js": { - "version": "9.3.1", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", - "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/buffer-equal-constant-time": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", - "license": "BSD-3-Clause" - }, - "node_modules/cjs-module-lexer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.2.0.tgz", - "integrity": "sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==", - "license": "MIT" - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "license": "MIT" - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/dotenv": { - "version": "17.2.3", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", - "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", - "license": "MIT" - }, - "node_modules/ecdsa-sig-formatter": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", - "license": "Apache-2.0", - "dependencies": { - "safe-buffer": "^5.0.1" - } - }, - "node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", - "license": "MIT" - }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "license": "MIT" - }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, - "node_modules/foreground-child": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", - "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", - "license": "ISC", - "dependencies": { - "cross-spawn": "^7.0.6", - "signal-exit": "^4.0.1" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "license": "MIT", - "dependencies": { - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": ">=12.20.0" - } - }, - "node_modules/forwarded-parse": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/forwarded-parse/-/forwarded-parse-2.1.2.tgz", - "integrity": "sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==", - "license": "MIT" - }, - "node_modules/gaxios": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.3.tgz", - "integrity": "sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==", - "license": "Apache-2.0", - "dependencies": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "node-fetch": "^3.3.2", - "rimraf": "^5.0.1" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/gcp-metadata": { - "version": "8.1.2", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", - "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", - "license": "Apache-2.0", - "dependencies": { - "gaxios": "^7.0.0", - "google-logging-utils": "^1.0.0", - "json-bigint": "^1.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/glob": { - "version": "10.4.5", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", - "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", - "license": "ISC", - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^3.1.2", - "minimatch": "^9.0.4", - "minipass": "^7.1.2", - "package-json-from-dist": "^1.0.0", - "path-scurry": "^1.11.1" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/google-auth-library": { - "version": "10.5.0", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.5.0.tgz", - "integrity": "sha512-7ABviyMOlX5hIVD60YOfHw4/CxOfBhyduaYB+wbFWCWoni4N7SLcV46hrVRktuBbZjFC9ONyqamZITN7q3n32w==", - "license": "Apache-2.0", - "dependencies": { - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "gaxios": "^7.0.0", - "gcp-metadata": "^8.0.0", - "google-logging-utils": "^1.0.0", - "gtoken": "^8.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/google-logging-utils": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.2.tgz", - "integrity": "sha512-YsFPGVgDFf4IzSwbwIR0iaFJQFmR5Jp7V1WuYSjuRgAm9yWqsMhKE9YPlL+wvFLnc/wMiFV4SQUD9Y/JMpxIxQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/gtoken": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-8.0.0.tgz", - "integrity": "sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==", - "license": "MIT", - "dependencies": { - "gaxios": "^7.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/https-proxy-agent": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/import-in-the-middle": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-2.0.4.tgz", - "integrity": "sha512-Al0kMpa0BqfvDnxjxGlab9vdQ0vTDs82TBKrD59X9jReUoPAzSGBb6vGDzMUMFBGyyDF03RpLT4oxGn6BpASzQ==", - "license": "Apache-2.0", - "dependencies": { - "acorn": "^8.15.0", - "acorn-import-attributes": "^1.9.5", - "cjs-module-lexer": "^2.2.0", - "module-details-from-path": "^1.0.4" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "license": "ISC" - }, - "node_modules/jackspeak": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", - "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - }, - "optionalDependencies": { - "@pkgjs/parseargs": "^0.11.0" - } - }, - "node_modules/json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "license": "MIT", - "dependencies": { - "bignumber.js": "^9.0.0" - } - }, - "node_modules/jwa": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", - "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", - "license": "MIT", - "dependencies": { - "buffer-equal-constant-time": "^1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", - "license": "MIT", - "dependencies": { - "jwa": "^2.0.0", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/lru-cache": { - "version": "10.4.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", - "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", - "license": "ISC" - }, - "node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "license": "ISC", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/module-details-from-path": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", - "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", - "license": "MIT" - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "license": "MIT", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, - "node_modules/package-json-from-dist": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", - "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", - "license": "BlueOak-1.0.0" - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-scurry": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", - "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^10.2.0", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" - }, - "engines": { - "node": ">=16 || 14 >=14.18" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/pg-int8": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", - "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", - "license": "ISC", - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/pg-protocol": { - "version": "1.10.3", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.10.3.tgz", - "integrity": "sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==", - "license": "MIT" - }, - "node_modules/pg-types": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", - "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", - "license": "MIT", - "dependencies": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-array": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", - "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-bytea": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", - "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-date": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", - "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-interval": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", - "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", - "license": "MIT", - "dependencies": { - "xtend": "^4.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-in-the-middle": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-8.0.1.tgz", - "integrity": "sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==", - "license": "MIT", - "dependencies": { - "debug": "^4.3.5", - "module-details-from-path": "^1.0.3" - }, - "engines": { - "node": ">=9.3.0 || >=8.10.0 <9.0.0" - } - }, - "node_modules/rimraf": { - "version": "5.0.10", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.10.tgz", - "integrity": "sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==", - "license": "ISC", - "dependencies": { - "glob": "^10.3.7" - }, - "bin": { - "rimraf": "dist/esm/bin.mjs" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "license": "MIT", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/string-width-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", - "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^6.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/undici-types": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", - "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", - "license": "MIT" - }, - "node_modules/web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs": { - "name": "wrap-ansi", - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/wrap-ansi-cjs/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/ws": { - "version": "8.18.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", - "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "license": "MIT", - "engines": { - "node": ">=0.4" - } - } - } -} diff --git a/sdks/js/google-genai/package.json b/sdks/js/google-genai/package.json deleted file mode 100644 index e052b28..0000000 --- a/sdks/js/google-genai/package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "@sentry-ai-sdks/google-genai", - "version": "1.0.0", - "description": "Google GenAI SDK integration tests for Sentry", - "dependencies": { - "@sentry/node": "10.34.0", - "@google/genai": "1.29.0", - "dotenv": "17.2.3" - } -} diff --git a/sdks/js/google-genai/setup.js b/sdks/js/google-genai/setup.js deleted file mode 100644 index ce2ffee..0000000 --- a/sdks/js/google-genai/setup.js +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Setup file for Google GenAI SDK tests - * - * Initializes Sentry with Google GenAI-specific integrations. - */ - -const Sentry = require("@sentry/node"); -const { createTransport } = require("@sentry/core"); -const { config } = require("dotenv"); -const { resolve } = require("path"); -const { createMockTransport } = require("../_test-utils/mock-transport.cjs"); - -// Load environment variables -config({ quiet: true, path: resolve(__dirname, "../../../.env") }); - -// Initialize Sentry. -// NOTE: AI integrations are auto-enabled in the Node.js SDK and -// therefore do not need to be configured explicitly. -Sentry.init({ - dsn: process.env.SENTRY_DSN || "https://public@127.0.0.1/1", - tracesSampleRate: 1.0, - transport: createMockTransport(createTransport), - sendDefaultPii: true, -}); - -module.exports = { Sentry }; diff --git a/sdks/js/langchain/cases/1-simple.js b/sdks/js/langchain/cases/1-simple.js deleted file mode 100644 index 639aa4e..0000000 --- a/sdks/js/langchain/cases/1-simple.js +++ /dev/null @@ -1,39 +0,0 @@ -/** - * 1-simple: Basic Completion - * - * Tests a simple chat completion request with LangChain SDK - * and verifies that Sentry captures the appropriate spans and AI monitoring data. - */ - -const { Sentry } = require("../setup"); -const { ChatOpenAI } = require("@langchain/openai"); -const { HumanMessage, SystemMessage } = require("@langchain/core/messages"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, system, prompt } = inputs; - - const chatModel = new ChatOpenAI({ - modelName: model, - apiKey: process.env.OPENAI_API_KEY, - }); - - const messages = [ - new SystemMessage(system), - new HumanMessage(prompt), - ]; - - const response = await chatModel.invoke(messages); - - const text = response.content; - - if (!text) { - throw new Error("No completion returned from LangChain"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${text}`); - } -} - -module.exports = runTestCase("1-simple", testLogic, Sentry); diff --git a/sdks/js/langchain/cases/10-binary-content-redaction.js b/sdks/js/langchain/cases/10-binary-content-redaction.js deleted file mode 100644 index 802d6dd..0000000 --- a/sdks/js/langchain/cases/10-binary-content-redaction.js +++ /dev/null @@ -1,60 +0,0 @@ -/** - * 10-binary-content-redaction: Binary Content Redaction - * - * Tests that binary image data is properly redacted in Sentry spans. - * Sends a message with a base64-encoded image and verifies that Sentry: - * - Captures the message structure - * - Redacts the binary content with "[Blob substitute]" marker - * - Does not send raw binary data to Sentry - */ - -const { Sentry } = require("../setup"); -const { ChatOpenAI } = require("@langchain/openai"); -const { HumanMessage } = require("@langchain/core/messages"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, image_type } = inputs; - - const chatModel = new ChatOpenAI({ - modelName: model, - apiKey: process.env.OPENAI_API_KEY, - }); - - // Create a small base64-encoded image (1x1 pixel PNG) - // This is a valid 1x1 transparent PNG image - const base64Image = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; - - // LangChain uses a specific format for multimodal messages with images - // The content is an array with text and image_url objects - const message = new HumanMessage({ - content: [ - { - type: "text", - text: "What color is this image? Answer in one word.", - }, - { - type: "image_url", - image_url: { - url: `data:image/${image_type};base64,${base64Image}`, - }, - }, - ], - }); - - const response = await chatModel.invoke([message]); - - const text = response.content; - - if (!text) { - throw new Error("No completion returned from LangChain"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${text}`); - console.log(` Sent message with base64-encoded ${image_type} image`); - } -} - -module.exports = runTestCase("10-binary-content-redaction", testLogic, Sentry); diff --git a/sdks/js/langchain/cases/2-multi-step.js b/sdks/js/langchain/cases/2-multi-step.js deleted file mode 100644 index 8e00498..0000000 --- a/sdks/js/langchain/cases/2-multi-step.js +++ /dev/null @@ -1,58 +0,0 @@ -/** - * 2-multi-step: Multi-step Conversation - * - * Tests a multi-step conversation with conversation history using LangChain SDK - * and verifies that Sentry captures all spans for both API calls. - */ - -const { Sentry } = require("../setup"); -const { ChatOpenAI } = require("@langchain/openai"); -const { HumanMessage, SystemMessage, AIMessage } = require("@langchain/core/messages"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, system, first_prompt, second_prompt } = inputs; - - const chatModel = new ChatOpenAI({ - modelName: model, - apiKey: process.env.OPENAI_API_KEY, - }); - - // First call - const firstMessages = [ - new SystemMessage(system), - new HumanMessage(first_prompt), - ]; - - const firstResponse = await chatModel.invoke(firstMessages); - const firstText = firstResponse.content; - - if (!firstText) { - throw new Error("No completion returned from LangChain (first call)"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` First response: ${firstText}`); - } - - // Second call with conversation history - const secondMessages = [ - new SystemMessage(system), - new HumanMessage(first_prompt), - new AIMessage(firstText), - new HumanMessage(second_prompt), - ]; - - const secondResponse = await chatModel.invoke(secondMessages); - const secondText = secondResponse.content; - - if (!secondText) { - throw new Error("No completion returned from LangChain (second call)"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Second response: ${secondText}`); - } -} - -module.exports = runTestCase("2-multi-step", testLogic, Sentry); diff --git a/sdks/js/langchain/cases/9-message-truncation.js b/sdks/js/langchain/cases/9-message-truncation.js deleted file mode 100644 index 70fe58a..0000000 --- a/sdks/js/langchain/cases/9-message-truncation.js +++ /dev/null @@ -1,58 +0,0 @@ -/** - * 9-message-truncation: Message Truncation - * - * Tests that large messages are properly truncated while preserving the original message count. - * Sends multiple large messages (~9KB each) and verifies that Sentry captures: - * - gen_ai.request.messages.original_length (the actual count of messages sent) - * - gen_ai.request.messages array (potentially truncated for telemetry) - * - The relationship: len(messages) <= original_length - */ - -const { Sentry } = require("../setup"); -const { ChatOpenAI } = require("@langchain/openai"); -const { HumanMessage, AIMessage } = require("@langchain/core/messages"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, message_size_kb, message_count } = inputs; - - const chatModel = new ChatOpenAI({ - modelName: model, - apiKey: process.env.OPENAI_API_KEY, - }); - - // Generate large message content (~9KB each) - const contentSize = message_size_kb * 1024; - const largeContent = "x".repeat(contentSize); - - // Create the messages array with large content - // Alternate between HumanMessage and AIMessage to simulate conversation - const messages = []; - for (let i = 0; i < message_count; i++) { - if (i % 2 === 0) { - messages.push(new HumanMessage(`Message ${i + 1}: ${largeContent}`)); - } else { - messages.push(new AIMessage(`Message ${i + 1}: ${largeContent}`)); - } - } - - // Ensure the last message is from user (required by OpenAI API) - if (messages[messages.length - 1] instanceof AIMessage) { - messages.push(new HumanMessage("Please summarize what we discussed.")); - } - - const response = await chatModel.invoke(messages); - - const text = response.content; - - if (!text) { - throw new Error("No completion returned from LangChain"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${text.substring(0, 100)}...`); - console.log(` Sent ${messages.length} messages with ~${message_size_kb}KB content each`); - } -} - -module.exports = runTestCase("9-message-truncation", testLogic, Sentry); diff --git a/sdks/js/langchain/config.json b/sdks/js/langchain/config.json deleted file mode 100644 index f634cbb..0000000 --- a/sdks/js/langchain/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "sdk_name": "langchain", - "framework_type": "low-level", - "overrides": {} -} diff --git a/sdks/js/langchain/package-lock.json b/sdks/js/langchain/package-lock.json deleted file mode 100644 index 1feeb0d..0000000 --- a/sdks/js/langchain/package-lock.json +++ /dev/null @@ -1,1302 +0,0 @@ -{ - "name": "@sentry-ai-sdks/langchain", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@sentry-ai-sdks/langchain", - "version": "1.0.0", - "dependencies": { - "@langchain/core": "1.1.0", - "@langchain/openai": "1.1.3", - "@sentry/node": "10.34.0", - "dotenv": "17.2.3" - } - }, - "node_modules/@apm-js-collab/code-transformer": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/@apm-js-collab/code-transformer/-/code-transformer-0.8.2.tgz", - "integrity": "sha512-YRjJjNq5KFSjDUoqu5pFUWrrsvGOxl6c3bu+uMFc9HNNptZ2rNU/TI2nLw4jnhQNtka972Ee2m3uqbvDQtPeCA==", - "license": "Apache-2.0" - }, - "node_modules/@apm-js-collab/tracing-hooks": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@apm-js-collab/tracing-hooks/-/tracing-hooks-0.3.1.tgz", - "integrity": "sha512-Vu1CbmPURlN5fTboVuKMoJjbO5qcq9fA5YXpskx3dXe/zTBvjODFoerw+69rVBlRLrJpwPqSDqEuJDEKIrTldw==", - "license": "Apache-2.0", - "dependencies": { - "@apm-js-collab/code-transformer": "^0.8.0", - "debug": "^4.4.1", - "module-details-from-path": "^1.0.4" - } - }, - "node_modules/@cfworker/json-schema": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", - "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", - "license": "MIT" - }, - "node_modules/@langchain/core": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.1.0.tgz", - "integrity": "sha512-yJ6JHcU9psjnQbzRFkXjIdNTA+3074dA+2pHdH8ewvQCSleSk6JcjkCMIb5+NASjeMoi1ZuntlLKVsNqF38YxA==", - "license": "MIT", - "dependencies": { - "@cfworker/json-schema": "^4.0.2", - "ansi-styles": "^5.0.0", - "camelcase": "6", - "decamelize": "1.2.0", - "js-tiktoken": "^1.0.12", - "langsmith": "^0.3.64", - "mustache": "^4.2.0", - "p-queue": "^6.6.2", - "p-retry": "^7.0.0", - "uuid": "^10.0.0", - "zod": "^3.25.76 || ^4" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@langchain/core/node_modules/p-retry": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-7.1.0.tgz", - "integrity": "sha512-xL4PiFRQa/f9L9ZvR4/gUCRNus4N8YX80ku8kv9Jqz+ZokkiZLM0bcvX0gm1F3PDi9SPRsww1BDsTWgE6Y1GLQ==", - "license": "MIT", - "dependencies": { - "is-network-error": "^1.1.0" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@langchain/openai": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-1.1.3.tgz", - "integrity": "sha512-p+xR+4HRms5Ozjf5miC6U2AYRyNVSTdO7AMBkMYs1Tp6DWHBd+mQ72H8Ogd2dKrPuS5UDJ5dbpI1fS+OrTbgQQ==", - "license": "MIT", - "dependencies": { - "js-tiktoken": "^1.0.12", - "openai": "^6.9.0", - "zod": "^3.25.76 || ^4" - }, - "engines": { - "node": ">=20" - }, - "peerDependencies": { - "@langchain/core": "^1.0.0" - } - }, - "node_modules/@opentelemetry/api": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", - "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", - "license": "Apache-2.0", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/api-logs": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.208.0.tgz", - "integrity": "sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api": "^1.3.0" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/context-async-hooks": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.4.0.tgz", - "integrity": "sha512-jn0phJ+hU7ZuvaoZE/8/Euw3gvHJrn2yi+kXrymwObEPVPjtwCmkvXDRQCWli+fCTTF/aSOtXaLr7CLIvv3LQg==", - "license": "Apache-2.0", - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/core": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.2.0.tgz", - "integrity": "sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/instrumentation": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.208.0.tgz", - "integrity": "sha512-Eju0L4qWcQS+oXxi6pgh7zvE2byogAkcsVv0OjHF/97iOz1N/aKE6etSGowYkie+YA1uo6DNwdSxaaNnLvcRlA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api-logs": "0.208.0", - "import-in-the-middle": "^2.0.0", - "require-in-the-middle": "^8.0.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-amqplib": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-amqplib/-/instrumentation-amqplib-0.55.0.tgz", - "integrity": "sha512-5ULoU8p+tWcQw5PDYZn8rySptGSLZHNX/7srqo2TioPnAAcvTy6sQFQXsNPrAnyRRtYGMetXVyZUy5OaX1+IfA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-connect": { - "version": "0.52.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-connect/-/instrumentation-connect-0.52.0.tgz", - "integrity": "sha512-GXPxfNB5szMbV3I9b7kNWSmQBoBzw7MT0ui6iU/p+NIzVx3a06Ri2cdQO7tG9EKb4aKSLmfX9Cw5cKxXqX6Ohg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0", - "@types/connect": "3.4.38" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-dataloader": { - "version": "0.26.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-dataloader/-/instrumentation-dataloader-0.26.0.tgz", - "integrity": "sha512-P2BgnFfTOarZ5OKPmYfbXfDFjQ4P9WkQ1Jji7yH5/WwB6Wm/knynAoA1rxbjWcDlYupFkyT0M1j6XLzDzy0aCA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-express": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-express/-/instrumentation-express-0.57.0.tgz", - "integrity": "sha512-HAdx/o58+8tSR5iW+ru4PHnEejyKrAy9fYFhlEI81o10nYxrGahnMAHWiSjhDC7UQSY3I4gjcPgSKQz4rm/asg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-fs": { - "version": "0.28.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.28.0.tgz", - "integrity": "sha512-FFvg8fq53RRXVBRHZViP+EMxMR03tqzEGpuq55lHNbVPyFklSVfQBN50syPhK5UYYwaStx0eyCtHtbRreusc5g==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-generic-pool": { - "version": "0.52.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-generic-pool/-/instrumentation-generic-pool-0.52.0.tgz", - "integrity": "sha512-ISkNcv5CM2IwvsMVL31Tl61/p2Zm2I2NAsYq5SSBgOsOndT0TjnptjufYVScCnD5ZLD1tpl4T3GEYULLYOdIdQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-graphql": { - "version": "0.56.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-graphql/-/instrumentation-graphql-0.56.0.tgz", - "integrity": "sha512-IPvNk8AFoVzTAM0Z399t34VDmGDgwT6rIqCUug8P9oAGerl2/PEIYMPOl/rerPGu+q8gSWdmbFSjgg7PDVRd3Q==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-hapi": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-hapi/-/instrumentation-hapi-0.55.0.tgz", - "integrity": "sha512-prqAkRf9e4eEpy4G3UcR32prKE8NLNlA90TdEU1UsghOTg0jUvs40Jz8LQWFEs5NbLbXHYGzB4CYVkCI8eWEVQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-http": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.208.0.tgz", - "integrity": "sha512-rhmK46DRWEbQQB77RxmVXGyjs6783crXCnFjYQj+4tDH/Kpv9Rbg3h2kaNyp5Vz2emF1f9HOQQvZoHzwMWOFZQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/instrumentation": "0.208.0", - "@opentelemetry/semantic-conventions": "^1.29.0", - "forwarded-parse": "2.1.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-ioredis": { - "version": "0.56.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.56.0.tgz", - "integrity": "sha512-XSWeqsd3rKSsT3WBz/JKJDcZD4QYElZEa0xVdX8f9dh4h4QgXhKRLorVsVkK3uXFbC2sZKAS2Ds+YolGwD83Dg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/redis-common": "^0.38.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-kafkajs": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-kafkajs/-/instrumentation-kafkajs-0.18.0.tgz", - "integrity": "sha512-KCL/1HnZN5zkUMgPyOxfGjLjbXjpd4odDToy+7c+UsthIzVLFf99LnfIBE8YSSrYE4+uS7OwJMhvhg3tWjqMBg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.30.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-knex": { - "version": "0.53.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-knex/-/instrumentation-knex-0.53.0.tgz", - "integrity": "sha512-xngn5cH2mVXFmiT1XfQ1aHqq1m4xb5wvU6j9lSgLlihJ1bXzsO543cpDwjrZm2nMrlpddBf55w8+bfS4qDh60g==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.33.1" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-koa": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-koa/-/instrumentation-koa-0.57.0.tgz", - "integrity": "sha512-3JS8PU/D5E3q295mwloU2v7c7/m+DyCqdu62BIzWt+3u9utjxC9QS7v6WmUNuoDN3RM+Q+D1Gpj13ERo+m7CGg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.36.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0" - } - }, - "node_modules/@opentelemetry/instrumentation-lru-memoizer": { - "version": "0.53.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.53.0.tgz", - "integrity": "sha512-LDwWz5cPkWWr0HBIuZUjslyvijljTwmwiItpMTHujaULZCxcYE9eU44Qf/pbVC8TulT0IhZi+RoGvHKXvNhysw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mongodb": { - "version": "0.61.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.61.0.tgz", - "integrity": "sha512-OV3i2DSoY5M/pmLk+68xr5RvkHU8DRB3DKMzYJdwDdcxeLs62tLbkmRyqJZsYf3Ht7j11rq35pHOWLuLzXL7pQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mongoose": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.55.0.tgz", - "integrity": "sha512-5afj0HfF6aM6Nlqgu6/PPHFk8QBfIe3+zF9FGpX76jWPS0/dujoEYn82/XcLSaW5LPUDW8sni+YeK0vTBNri+w==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mysql": { - "version": "0.54.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql/-/instrumentation-mysql-0.54.0.tgz", - "integrity": "sha512-bqC1YhnwAeWmRzy1/Xf9cDqxNG2d/JDkaxnqF5N6iJKN1eVWI+vg7NfDkf52/Nggp3tl1jcC++ptC61BD6738A==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@types/mysql": "2.15.27" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mysql2": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql2/-/instrumentation-mysql2-0.55.0.tgz", - "integrity": "sha512-0cs8whQG55aIi20gnK8B7cco6OK6N+enNhW0p5284MvqJ5EPi+I1YlWsWXgzv/V2HFirEejkvKiI4Iw21OqDWg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.33.0", - "@opentelemetry/sql-common": "^0.41.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-pg": { - "version": "0.61.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.61.0.tgz", - "integrity": "sha512-UeV7KeTnRSM7ECHa3YscoklhUtTQPs6V6qYpG283AB7xpnPGCUCUfECFT9jFg6/iZOQTt3FHkB1wGTJCNZEvPw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.34.0", - "@opentelemetry/sql-common": "^0.41.2", - "@types/pg": "8.15.6", - "@types/pg-pool": "2.0.6" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-redis": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-redis/-/instrumentation-redis-0.57.0.tgz", - "integrity": "sha512-bCxTHQFXzrU3eU1LZnOZQ3s5LURxQPDlU3/upBzlWY77qOI1GZuGofazj3jtzjctMJeBEJhNwIFEgRPBX1kp/Q==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/redis-common": "^0.38.2", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-tedious": { - "version": "0.27.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-tedious/-/instrumentation-tedious-0.27.0.tgz", - "integrity": "sha512-jRtyUJNZppPBjPae4ZjIQ2eqJbcRaRfJkr0lQLHFmOU/no5A6e9s1OHLd5XZyZoBJ/ymngZitanyRRA5cniseA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@types/tedious": "^4.0.14" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-undici": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.19.0.tgz", - "integrity": "sha512-Pst/RhR61A2OoZQZkn6OLpdVpXp6qn3Y92wXa6umfJe9rV640r4bc6SWvw4pPN6DiQqPu2c8gnSSZPDtC6JlpQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.24.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.7.0" - } - }, - "node_modules/@opentelemetry/redis-common": { - "version": "0.38.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/redis-common/-/redis-common-0.38.2.tgz", - "integrity": "sha512-1BCcU93iwSRZvDAgwUxC/DV4T/406SkMfxGqu5ojc3AvNI+I9GhV7v0J1HljsczuuhcnFLYqD5VmwVXfCGHzxA==", - "license": "Apache-2.0", - "engines": { - "node": "^18.19.0 || >=20.6.0" - } - }, - "node_modules/@opentelemetry/resources": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.2.0.tgz", - "integrity": "sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/sdk-trace-base": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.2.0.tgz", - "integrity": "sha512-xWQgL0Bmctsalg6PaXExmzdedSp3gyKV8mQBwK/j9VGdCDu2fmXIb2gAehBKbkXCpJ4HPkgv3QfoJWRT4dHWbw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/resources": "2.2.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/semantic-conventions": { - "version": "1.38.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.38.0.tgz", - "integrity": "sha512-kocjix+/sSggfJhwXqClZ3i9Y/MI0fp7b+g7kCRm6psy2dsf8uApTRclwG18h8Avm7C9+fnt+O36PspJ/OzoWg==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/@opentelemetry/sql-common": { - "version": "0.41.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/sql-common/-/sql-common-0.41.2.tgz", - "integrity": "sha512-4mhWm3Z8z+i508zQJ7r6Xi7y4mmoJpdvH0fZPFRkWrdp5fq7hhZ2HhYokEOLkfqSMgPR4Z9EyB3DBkbKGOqZiQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.1.0" - } - }, - "node_modules/@prisma/instrumentation": { - "version": "6.19.0", - "resolved": "https://registry.npmjs.org/@prisma/instrumentation/-/instrumentation-6.19.0.tgz", - "integrity": "sha512-QcuYy25pkXM8BJ37wVFBO7Zh34nyRV1GOb2n3lPkkbRYfl4hWl3PTcImP41P0KrzVXfa/45p6eVCos27x3exIg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": ">=0.52.0 <1" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.8" - } - }, - "node_modules/@sentry/core": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/core/-/core-10.34.0.tgz", - "integrity": "sha512-4FFpYBMf0VFdPcsr4grDYDOR87mRu6oCfb51oQjU/Pndmty7UgYo0Bst3LEC/8v0SpytBtzXq+Wx/fkwulBesg==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@sentry/node": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/node/-/node-10.34.0.tgz", - "integrity": "sha512-bEOyH97HuVtWZYAZ5mp0NhYNc+n6QCfiKuLee2P75n2kt4cIPTGvLOSdUwwjllf795uOdKZJuM1IUN0W+YMcVg==", - "license": "MIT", - "dependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^2.2.0", - "@opentelemetry/core": "^2.2.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/instrumentation-amqplib": "0.55.0", - "@opentelemetry/instrumentation-connect": "0.52.0", - "@opentelemetry/instrumentation-dataloader": "0.26.0", - "@opentelemetry/instrumentation-express": "0.57.0", - "@opentelemetry/instrumentation-fs": "0.28.0", - "@opentelemetry/instrumentation-generic-pool": "0.52.0", - "@opentelemetry/instrumentation-graphql": "0.56.0", - "@opentelemetry/instrumentation-hapi": "0.55.0", - "@opentelemetry/instrumentation-http": "0.208.0", - "@opentelemetry/instrumentation-ioredis": "0.56.0", - "@opentelemetry/instrumentation-kafkajs": "0.18.0", - "@opentelemetry/instrumentation-knex": "0.53.0", - "@opentelemetry/instrumentation-koa": "0.57.0", - "@opentelemetry/instrumentation-lru-memoizer": "0.53.0", - "@opentelemetry/instrumentation-mongodb": "0.61.0", - "@opentelemetry/instrumentation-mongoose": "0.55.0", - "@opentelemetry/instrumentation-mysql": "0.54.0", - "@opentelemetry/instrumentation-mysql2": "0.55.0", - "@opentelemetry/instrumentation-pg": "0.61.0", - "@opentelemetry/instrumentation-redis": "0.57.0", - "@opentelemetry/instrumentation-tedious": "0.27.0", - "@opentelemetry/instrumentation-undici": "0.19.0", - "@opentelemetry/resources": "^2.2.0", - "@opentelemetry/sdk-trace-base": "^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0", - "@prisma/instrumentation": "6.19.0", - "@sentry/core": "10.34.0", - "@sentry/node-core": "10.34.0", - "@sentry/opentelemetry": "10.34.0", - "import-in-the-middle": "^2.0.1", - "minimatch": "^9.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@sentry/node-core": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/node-core/-/node-core-10.34.0.tgz", - "integrity": "sha512-FrGfC8GzD1cnZDO3zwQ4cjyoY1ZwNHvZbXSvXRYxpjhXidZhvaPurjgLRSB0xGaFgoemmOp1ufsx/w6fQOGA6Q==", - "license": "MIT", - "dependencies": { - "@apm-js-collab/tracing-hooks": "^0.3.1", - "@sentry/core": "10.34.0", - "@sentry/opentelemetry": "10.34.0", - "import-in-the-middle": "^2.0.1" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/core": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/instrumentation": ">=0.57.1 <1", - "@opentelemetry/resources": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0" - } - }, - "node_modules/@sentry/opentelemetry": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/opentelemetry/-/opentelemetry-10.34.0.tgz", - "integrity": "sha512-uKuULBOmdVu3bYdD8doMLqKgN0PP3WWtI7Shu11P9PVrhSNT4U9yM9Z6v1aFlQcbrgyg3LynZuXs8lyjt90UbA==", - "license": "MIT", - "dependencies": { - "@sentry/core": "10.34.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/core": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0" - } - }, - "node_modules/@types/connect": { - "version": "3.4.38", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", - "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/mysql": { - "version": "2.15.27", - "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.27.tgz", - "integrity": "sha512-YfWiV16IY0OeBfBCk8+hXKmdTKrKlwKN1MNKAPBu5JYxLwBEZl7QzeEpGnlZb3VMGJrrGmB84gXiH+ofs/TezA==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/node": { - "version": "24.10.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", - "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", - "license": "MIT", - "dependencies": { - "undici-types": "~7.16.0" - } - }, - "node_modules/@types/pg": { - "version": "8.15.6", - "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.15.6.tgz", - "integrity": "sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "pg-protocol": "*", - "pg-types": "^2.2.0" - } - }, - "node_modules/@types/pg-pool": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@types/pg-pool/-/pg-pool-2.0.6.tgz", - "integrity": "sha512-TaAUE5rq2VQYxab5Ts7WZhKNmuN78Q6PiFonTDdpbx8a1H0M1vhy3rhiMjl+e2iHmogyMw7jZF4FrE6eJUy5HQ==", - "license": "MIT", - "dependencies": { - "@types/pg": "*" - } - }, - "node_modules/@types/retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", - "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", - "license": "MIT" - }, - "node_modules/@types/tedious": { - "version": "4.0.14", - "resolved": "https://registry.npmjs.org/@types/tedious/-/tedious-4.0.14.tgz", - "integrity": "sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/uuid": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", - "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", - "license": "MIT" - }, - "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", - "license": "MIT", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-import-attributes": { - "version": "1.9.5", - "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", - "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", - "license": "MIT", - "peerDependencies": { - "acorn": "^8" - } - }, - "node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "license": "MIT" - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/camelcase": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", - "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/chalk/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/cjs-module-lexer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.2.0.tgz", - "integrity": "sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==", - "license": "MIT" - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "license": "MIT" - }, - "node_modules/console-table-printer": { - "version": "2.15.0", - "resolved": "https://registry.npmjs.org/console-table-printer/-/console-table-printer-2.15.0.tgz", - "integrity": "sha512-SrhBq4hYVjLCkBVOWaTzceJalvn5K1Zq5aQA6wXC/cYjI3frKWNPEMK3sZsJfNNQApvCQmgBcc13ZKmFj8qExw==", - "license": "MIT", - "dependencies": { - "simple-wcswidth": "^1.1.2" - } - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/dotenv": { - "version": "17.2.3", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", - "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/eventemitter3": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", - "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", - "license": "MIT" - }, - "node_modules/forwarded-parse": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/forwarded-parse/-/forwarded-parse-2.1.2.tgz", - "integrity": "sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==", - "license": "MIT" - }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/import-in-the-middle": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-2.0.4.tgz", - "integrity": "sha512-Al0kMpa0BqfvDnxjxGlab9vdQ0vTDs82TBKrD59X9jReUoPAzSGBb6vGDzMUMFBGyyDF03RpLT4oxGn6BpASzQ==", - "license": "Apache-2.0", - "dependencies": { - "acorn": "^8.15.0", - "acorn-import-attributes": "^1.9.5", - "cjs-module-lexer": "^2.2.0", - "module-details-from-path": "^1.0.4" - } - }, - "node_modules/is-network-error": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.0.tgz", - "integrity": "sha512-6oIwpsgRfnDiyEDLMay/GqCl3HoAtH5+RUKW29gYkL0QA+ipzpDLA16yQs7/RHCSu+BwgbJaOUqa4A99qNVQVw==", - "license": "MIT", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/js-tiktoken": { - "version": "1.0.21", - "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", - "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", - "license": "MIT", - "dependencies": { - "base64-js": "^1.5.1" - } - }, - "node_modules/langsmith": { - "version": "0.3.79", - "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.3.79.tgz", - "integrity": "sha512-j5uiAsyy90zxlxaMuGjb7EdcL51Yx61SpKfDOI1nMPBbemGju+lf47he4e59Hp5K63CY8XWgFP42WeZ+zuIU4Q==", - "license": "MIT", - "dependencies": { - "@types/uuid": "^10.0.0", - "chalk": "^4.1.2", - "console-table-printer": "^2.12.1", - "p-queue": "^6.6.2", - "p-retry": "4", - "semver": "^7.6.3", - "uuid": "^10.0.0" - }, - "peerDependencies": { - "@opentelemetry/api": "*", - "@opentelemetry/exporter-trace-otlp-proto": "*", - "@opentelemetry/sdk-trace-base": "*", - "openai": "*" - }, - "peerDependenciesMeta": { - "@opentelemetry/api": { - "optional": true - }, - "@opentelemetry/exporter-trace-otlp-proto": { - "optional": true - }, - "@opentelemetry/sdk-trace-base": { - "optional": true - }, - "openai": { - "optional": true - } - } - }, - "node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/module-details-from-path": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", - "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", - "license": "MIT" - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/mustache": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", - "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", - "license": "MIT", - "bin": { - "mustache": "bin/mustache" - } - }, - "node_modules/openai": { - "version": "6.9.1", - "resolved": "https://registry.npmjs.org/openai/-/openai-6.9.1.tgz", - "integrity": "sha512-vQ5Rlt0ZgB3/BNmTa7bIijYFhz3YBceAA3Z4JuoMSBftBF9YqFHIEhZakSs+O/Ad7EaoEimZvHxD5ylRjN11Lg==", - "license": "Apache-2.0", - "bin": { - "openai": "bin/cli" - }, - "peerDependencies": { - "ws": "^8.18.0", - "zod": "^3.25 || ^4.0" - }, - "peerDependenciesMeta": { - "ws": { - "optional": true - }, - "zod": { - "optional": true - } - } - }, - "node_modules/p-finally": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/p-queue": { - "version": "6.6.2", - "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", - "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", - "license": "MIT", - "dependencies": { - "eventemitter3": "^4.0.4", - "p-timeout": "^3.2.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-retry": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", - "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", - "license": "MIT", - "dependencies": { - "@types/retry": "0.12.0", - "retry": "^0.13.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-timeout": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", - "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", - "license": "MIT", - "dependencies": { - "p-finally": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pg-int8": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", - "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", - "license": "ISC", - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/pg-protocol": { - "version": "1.10.3", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.10.3.tgz", - "integrity": "sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==", - "license": "MIT" - }, - "node_modules/pg-types": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", - "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", - "license": "MIT", - "dependencies": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-array": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", - "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-bytea": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", - "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-date": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", - "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-interval": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", - "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", - "license": "MIT", - "dependencies": { - "xtend": "^4.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-in-the-middle": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-8.0.1.tgz", - "integrity": "sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==", - "license": "MIT", - "dependencies": { - "debug": "^4.3.5", - "module-details-from-path": "^1.0.3" - }, - "engines": { - "node": ">=9.3.0 || >=8.10.0 <9.0.0" - } - }, - "node_modules/retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/semver": { - "version": "7.7.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", - "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/simple-wcswidth": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/simple-wcswidth/-/simple-wcswidth-1.1.2.tgz", - "integrity": "sha512-j7piyCjAeTDSjzTSQ7DokZtMNwNlEAyxqSZeCS+CXH7fJ4jx3FuJ/mTW3mE+6JLs4VJBbcll0Kjn+KXI5t21Iw==", - "license": "MIT" - }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/undici-types": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", - "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", - "license": "MIT" - }, - "node_modules/uuid": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", - "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "license": "MIT", - "engines": { - "node": ">=0.4" - } - }, - "node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - } - } -} diff --git a/sdks/js/langchain/package.json b/sdks/js/langchain/package.json deleted file mode 100644 index 5d2e8ad..0000000 --- a/sdks/js/langchain/package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "@sentry-ai-sdks/langchain", - "version": "1.0.0", - "description": "LangChain SDK integration tests for Sentry", - "dependencies": { - "@sentry/node": "10.34.0", - "@langchain/core": "1.1.0", - "@langchain/openai": "1.1.3", - "dotenv": "17.2.3" - } -} diff --git a/sdks/js/langchain/setup.js b/sdks/js/langchain/setup.js deleted file mode 100644 index 0ac509c..0000000 --- a/sdks/js/langchain/setup.js +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Setup file for LangChain SDK tests - * - * Initializes Sentry with LangChain-specific integrations. - */ - -const Sentry = require("@sentry/node"); -const { createTransport } = require("@sentry/core"); -const { config } = require("dotenv"); -const { resolve } = require("path"); -const { createMockTransport } = require("../_test-utils/mock-transport.cjs"); - -// Load environment variables -config({ quiet: true, path: resolve(__dirname, "../../../.env") }); - -// Initialize Sentry. -// NOTE: AI integrations are auto-enabled in the Node.js SDK and -// therefore do not need to be configured explicitly. -Sentry.init({ - dsn: process.env.SENTRY_DSN || "https://public@127.0.0.1/1", - tracesSampleRate: 1.0, - transport: createMockTransport(createTransport), - sendDefaultPii: true, -}); - -module.exports = { Sentry }; diff --git a/sdks/js/langgraph/cases/1-simple.js b/sdks/js/langgraph/cases/1-simple.js deleted file mode 100644 index 9683533..0000000 --- a/sdks/js/langgraph/cases/1-simple.js +++ /dev/null @@ -1,45 +0,0 @@ -/** - * 1-simple: Basic Completion - * - * Tests a simple agent workflow request with LangGraph SDK - * and verifies that Sentry captures the appropriate spans and AI monitoring data. - */ - -const { Sentry } = require("../setup"); -const { ChatOpenAI } = require("@langchain/openai"); -const { createReactAgent } = require("@langchain/langgraph/prebuilt"); -const { HumanMessage, SystemMessage } = require("@langchain/core/messages"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, system, prompt } = inputs; - - // Create LLM instance - const llm = new ChatOpenAI({ - modelName: model, - apiKey: process.env.OPENAI_API_KEY, - }); - - // Create a simple react agent with no tools - const agent = createReactAgent({ llm, tools: [] }); - - // Invoke the agent with system and user messages - const result = await agent.invoke({ - messages: [new SystemMessage(system), new HumanMessage(prompt)], - }); - - // Extract the AI's response from the result - const messages = result.messages; - const lastMessage = messages[messages.length - 1]; - const text = lastMessage.content; - - if (!text) { - throw new Error("No completion returned from LangGraph"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${text}`); - } -} - -module.exports = runTestCase("1-simple", testLogic, Sentry); diff --git a/sdks/js/langgraph/cases/10-binary-content-redaction.js b/sdks/js/langgraph/cases/10-binary-content-redaction.js deleted file mode 100644 index b477226..0000000 --- a/sdks/js/langgraph/cases/10-binary-content-redaction.js +++ /dev/null @@ -1,79 +0,0 @@ -/** - * 10-binary-content-redaction: Binary Content Redaction Test - * - * Tests that when binary data (such as images) is sent to an LLM via LangGraph, - * Sentry correctly redacts the binary content in the captured span data and - * replaces it with a substitute marker ("[Blob substitute]"). - */ - -const { Sentry } = require("../setup"); -const { ChatOpenAI } = require("@langchain/openai"); -const { createReactAgent } = require("@langchain/langgraph/prebuilt"); -const { HumanMessage } = require("@langchain/core/messages"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -/** - * Creates a minimal valid PNG image (10x10 red square). - * Returns base64-encoded PNG data. - */ -function createMinimalPng() { - // Minimal 10x10 red PNG image (base64) - // This is a pre-generated minimal valid PNG to avoid needing canvas/sharp dependencies - const minimalPngBase64 = - "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8z8BQz0AEYBxVSF+FABJADveWkH6oAAAAAElFTkSuQmCC"; - - return minimalPngBase64; -} - -async function testLogic(inputs) { - const { model, image_type } = inputs; - - // Create LLM instance - const llm = new ChatOpenAI({ - modelName: model, - apiKey: process.env.OPENAI_API_KEY, - }); - - // Create a simple react agent with no tools - const agent = createReactAgent({ llm, tools: [] }); - - // Create binary image data - const base64Image = createMinimalPng(); - - // Create message with image content using LangChain's multimodal format - const message = new HumanMessage({ - content: [ - { - type: "image_url", - image_url: { url: `data:image/${image_type};base64,${base64Image}` }, - }, - { - type: "text", - text: "What color is this image? Answer in one word.", - }, - ], - }); - - // Invoke the agent with image message - const result = await agent.invoke({ - messages: [message], - }); - - // Extract the AI's response from the result - const resultMessages = result.messages; - const lastMessage = resultMessages[resultMessages.length - 1]; - const text = lastMessage.content; - - if (!text) { - throw new Error("No completion returned from LangGraph"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${text}`); - console.log( - ` Sent image with ${base64Image.length} characters of base64 data` - ); - } -} - -module.exports = runTestCase("10-binary-content-redaction", testLogic, Sentry); diff --git a/sdks/js/langgraph/cases/2-multi-step.js b/sdks/js/langgraph/cases/2-multi-step.js deleted file mode 100644 index 3c3703f..0000000 --- a/sdks/js/langgraph/cases/2-multi-step.js +++ /dev/null @@ -1,70 +0,0 @@ -/** - * 2-multi-step: Multi-step Conversation - * - * Tests a multi-step conversation with conversation history using LangGraph SDK - * and verifies that Sentry captures all spans for both API calls. - */ - -const { Sentry } = require("../setup"); -const { ChatOpenAI } = require("@langchain/openai"); -const { createReactAgent } = require("@langchain/langgraph/prebuilt"); -const { - HumanMessage, - SystemMessage, - AIMessage, -} = require("@langchain/core/messages"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, system, first_prompt, second_prompt } = inputs; - - // Create LLM instance - const llm = new ChatOpenAI({ - modelName: model, - apiKey: process.env.OPENAI_API_KEY, - }); - - // Create a simple react agent with no tools - const agent = createReactAgent({ llm, tools: [] }); - - // First call - const firstResult = await agent.invoke({ - messages: [new SystemMessage(system), new HumanMessage(first_prompt)], - }); - - const firstMessages = firstResult.messages; - const firstLastMessage = firstMessages[firstMessages.length - 1]; - const firstText = firstLastMessage.content; - - if (!firstText) { - throw new Error("No completion returned from LangGraph (first call)"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` First response: ${firstText}`); - } - - // Second call with conversation history - const secondResult = await agent.invoke({ - messages: [ - new SystemMessage(system), - new HumanMessage(first_prompt), - new AIMessage(firstText), - new HumanMessage(second_prompt), - ], - }); - - const secondMessages = secondResult.messages; - const secondLastMessage = secondMessages[secondMessages.length - 1]; - const secondText = secondLastMessage.content; - - if (!secondText) { - throw new Error("No completion returned from LangGraph (second call)"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Second response: ${secondText}`); - } -} - -module.exports = runTestCase("2-multi-step", testLogic, Sentry); diff --git a/sdks/js/langgraph/cases/9-message-truncation.js b/sdks/js/langgraph/cases/9-message-truncation.js deleted file mode 100644 index 76c8beb..0000000 --- a/sdks/js/langgraph/cases/9-message-truncation.js +++ /dev/null @@ -1,62 +0,0 @@ -/** - * 9-message-truncation: Message Truncation - * - * Tests that large messages are properly truncated while preserving the original message count. - * Sends multiple large messages (~9KB each) and verifies that Sentry captures: - * - gen_ai.request.messages.original_length (the actual count of messages sent) - * - gen_ai.request.messages array (potentially truncated for telemetry) - * - The relationship: len(messages) <= original_length - */ - -const { Sentry } = require("../setup"); -const { ChatOpenAI } = require("@langchain/openai"); -const { createReactAgent } = require("@langchain/langgraph/prebuilt"); -const { HumanMessage } = require("@langchain/core/messages"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, message_size_kb, message_count } = inputs; - - // Create LLM instance - const llm = new ChatOpenAI({ - modelName: model, - apiKey: process.env.OPENAI_API_KEY, - }); - - // Create a simple react agent with no tools - const agent = createReactAgent({ llm, tools: [] }); - - // Generate large message content (~9KB each) - // Using approximately 1 character per byte for ASCII - const contentSize = message_size_kb * 1024; - const largeContent = "A".repeat(contentSize); - - // Build messages array with multiple large messages - const messages = []; - for (let i = 0; i < message_count; i++) { - messages.push(new HumanMessage(`Message ${i + 1}: ${largeContent}`)); - } - - // Invoke the agent with large messages - const result = await agent.invoke({ - messages: messages, - }); - - // Extract the AI's response from the result - const resultMessages = result.messages; - const lastMessage = resultMessages[resultMessages.length - 1]; - const text = lastMessage.content; - - if (!text) { - throw new Error("No completion returned from LangGraph"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${text}`); - console.log( - ` Sent ${messages.length} messages with ~${message_size_kb}KB each` - ); - } -} - -module.exports = runTestCase("9-message-truncation", testLogic, Sentry); diff --git a/sdks/js/langgraph/config.json b/sdks/js/langgraph/config.json deleted file mode 100644 index 62b271b..0000000 --- a/sdks/js/langgraph/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "sdk_name": "langgraph", - "framework_type": "low-level", - "overrides": {} -} diff --git a/sdks/js/langgraph/package-lock.json b/sdks/js/langgraph/package-lock.json deleted file mode 100644 index a0a6b23..0000000 --- a/sdks/js/langgraph/package-lock.json +++ /dev/null @@ -1,1775 +0,0 @@ -{ - "name": "@sentry-ai-sdks/langgraph", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@sentry-ai-sdks/langgraph", - "version": "1.0.0", - "dependencies": { - "@langchain/core": "0.3.39", - "@langchain/langgraph": "0.2.31", - "@langchain/openai": "0.4.6", - "@sentry/node": "10.34.0", - "dotenv": "17.2.3" - } - }, - "node_modules/@apm-js-collab/code-transformer": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/@apm-js-collab/code-transformer/-/code-transformer-0.8.2.tgz", - "integrity": "sha512-YRjJjNq5KFSjDUoqu5pFUWrrsvGOxl6c3bu+uMFc9HNNptZ2rNU/TI2nLw4jnhQNtka972Ee2m3uqbvDQtPeCA==", - "license": "Apache-2.0" - }, - "node_modules/@apm-js-collab/tracing-hooks": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@apm-js-collab/tracing-hooks/-/tracing-hooks-0.3.1.tgz", - "integrity": "sha512-Vu1CbmPURlN5fTboVuKMoJjbO5qcq9fA5YXpskx3dXe/zTBvjODFoerw+69rVBlRLrJpwPqSDqEuJDEKIrTldw==", - "license": "Apache-2.0", - "dependencies": { - "@apm-js-collab/code-transformer": "^0.8.0", - "debug": "^4.4.1", - "module-details-from-path": "^1.0.4" - } - }, - "node_modules/@cfworker/json-schema": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", - "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", - "license": "MIT" - }, - "node_modules/@langchain/core": { - "version": "0.3.39", - "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.3.39.tgz", - "integrity": "sha512-muXs4asy1A7qDtcdznxqyBfxf4N6qxofY/S0c95vbsWa0r9YAE2PttHIjcuxSy1q2jUiTkpCcgFEjNJRQRVhEw==", - "license": "MIT", - "dependencies": { - "@cfworker/json-schema": "^4.0.2", - "ansi-styles": "^5.0.0", - "camelcase": "6", - "decamelize": "1.2.0", - "js-tiktoken": "^1.0.12", - "langsmith": ">=0.2.8 <0.4.0", - "mustache": "^4.2.0", - "p-queue": "^6.6.2", - "p-retry": "4", - "uuid": "^10.0.0", - "zod": "^3.22.4", - "zod-to-json-schema": "^3.22.3" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@langchain/langgraph": { - "version": "0.2.31", - "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-0.2.31.tgz", - "integrity": "sha512-/otJC3/P3Pt58eVZz1gxC3sBiC0N0HhOaAbOBKxckskhayBO6OC6ZDHtH9a+rxEIlreBoninR1/At1Gj/3liFA==", - "license": "MIT", - "dependencies": { - "@langchain/langgraph-checkpoint": "~0.0.13", - "@langchain/langgraph-sdk": "~0.0.21", - "uuid": "^10.0.0", - "zod": "^3.23.8" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@langchain/core": ">=0.2.36 <0.3.0 || >=0.3.9 < 0.4.0" - } - }, - "node_modules/@langchain/langgraph-checkpoint": { - "version": "0.0.18", - "resolved": "https://registry.npmjs.org/@langchain/langgraph-checkpoint/-/langgraph-checkpoint-0.0.18.tgz", - "integrity": "sha512-IS7zJj36VgY+4pf8ZjsVuUWef7oTwt1y9ylvwu0aLuOn1d0fg05Om9DLm3v2GZ2Df6bhLV1kfWAM0IAl9O5rQQ==", - "license": "MIT", - "dependencies": { - "uuid": "^10.0.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@langchain/core": ">=0.2.31 <0.4.0" - } - }, - "node_modules/@langchain/langgraph-sdk": { - "version": "0.0.112", - "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-0.0.112.tgz", - "integrity": "sha512-/9W5HSWCqYgwma6EoOspL4BGYxGxeJP6lIquPSF4FA0JlKopaUv58ucZC3vAgdJyCgg6sorCIV/qg7SGpEcCLw==", - "license": "MIT", - "dependencies": { - "@types/json-schema": "^7.0.15", - "p-queue": "^6.6.2", - "p-retry": "4", - "uuid": "^9.0.0" - }, - "peerDependencies": { - "@langchain/core": ">=0.2.31 <0.4.0", - "react": "^18 || ^19", - "react-dom": "^18 || ^19" - }, - "peerDependenciesMeta": { - "@langchain/core": { - "optional": true - }, - "react": { - "optional": true - }, - "react-dom": { - "optional": true - } - } - }, - "node_modules/@langchain/langgraph-sdk/node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/@langchain/openai": { - "version": "0.4.6", - "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.4.6.tgz", - "integrity": "sha512-cL+aGcglJCDsyfbsgi94C+Mw3o0Tlq834Gn0SiwA8aw4y8FYsDxX629nD+7sD1erCDu9AEpcmb5+1M40NjG+gg==", - "license": "MIT", - "dependencies": { - "js-tiktoken": "^1.0.12", - "openai": "^4.87.3", - "zod": "^3.22.4", - "zod-to-json-schema": "^3.22.3" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@langchain/core": ">=0.3.39 <0.4.0" - } - }, - "node_modules/@opentelemetry/api": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", - "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", - "license": "Apache-2.0", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/api-logs": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.208.0.tgz", - "integrity": "sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api": "^1.3.0" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/context-async-hooks": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.4.0.tgz", - "integrity": "sha512-jn0phJ+hU7ZuvaoZE/8/Euw3gvHJrn2yi+kXrymwObEPVPjtwCmkvXDRQCWli+fCTTF/aSOtXaLr7CLIvv3LQg==", - "license": "Apache-2.0", - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/core": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.2.0.tgz", - "integrity": "sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/instrumentation": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.208.0.tgz", - "integrity": "sha512-Eju0L4qWcQS+oXxi6pgh7zvE2byogAkcsVv0OjHF/97iOz1N/aKE6etSGowYkie+YA1uo6DNwdSxaaNnLvcRlA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api-logs": "0.208.0", - "import-in-the-middle": "^2.0.0", - "require-in-the-middle": "^8.0.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-amqplib": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-amqplib/-/instrumentation-amqplib-0.55.0.tgz", - "integrity": "sha512-5ULoU8p+tWcQw5PDYZn8rySptGSLZHNX/7srqo2TioPnAAcvTy6sQFQXsNPrAnyRRtYGMetXVyZUy5OaX1+IfA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-connect": { - "version": "0.52.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-connect/-/instrumentation-connect-0.52.0.tgz", - "integrity": "sha512-GXPxfNB5szMbV3I9b7kNWSmQBoBzw7MT0ui6iU/p+NIzVx3a06Ri2cdQO7tG9EKb4aKSLmfX9Cw5cKxXqX6Ohg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0", - "@types/connect": "3.4.38" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-dataloader": { - "version": "0.26.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-dataloader/-/instrumentation-dataloader-0.26.0.tgz", - "integrity": "sha512-P2BgnFfTOarZ5OKPmYfbXfDFjQ4P9WkQ1Jji7yH5/WwB6Wm/knynAoA1rxbjWcDlYupFkyT0M1j6XLzDzy0aCA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-express": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-express/-/instrumentation-express-0.57.0.tgz", - "integrity": "sha512-HAdx/o58+8tSR5iW+ru4PHnEejyKrAy9fYFhlEI81o10nYxrGahnMAHWiSjhDC7UQSY3I4gjcPgSKQz4rm/asg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-fs": { - "version": "0.28.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.28.0.tgz", - "integrity": "sha512-FFvg8fq53RRXVBRHZViP+EMxMR03tqzEGpuq55lHNbVPyFklSVfQBN50syPhK5UYYwaStx0eyCtHtbRreusc5g==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-generic-pool": { - "version": "0.52.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-generic-pool/-/instrumentation-generic-pool-0.52.0.tgz", - "integrity": "sha512-ISkNcv5CM2IwvsMVL31Tl61/p2Zm2I2NAsYq5SSBgOsOndT0TjnptjufYVScCnD5ZLD1tpl4T3GEYULLYOdIdQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-graphql": { - "version": "0.56.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-graphql/-/instrumentation-graphql-0.56.0.tgz", - "integrity": "sha512-IPvNk8AFoVzTAM0Z399t34VDmGDgwT6rIqCUug8P9oAGerl2/PEIYMPOl/rerPGu+q8gSWdmbFSjgg7PDVRd3Q==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-hapi": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-hapi/-/instrumentation-hapi-0.55.0.tgz", - "integrity": "sha512-prqAkRf9e4eEpy4G3UcR32prKE8NLNlA90TdEU1UsghOTg0jUvs40Jz8LQWFEs5NbLbXHYGzB4CYVkCI8eWEVQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-http": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.208.0.tgz", - "integrity": "sha512-rhmK46DRWEbQQB77RxmVXGyjs6783crXCnFjYQj+4tDH/Kpv9Rbg3h2kaNyp5Vz2emF1f9HOQQvZoHzwMWOFZQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/instrumentation": "0.208.0", - "@opentelemetry/semantic-conventions": "^1.29.0", - "forwarded-parse": "2.1.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-ioredis": { - "version": "0.56.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.56.0.tgz", - "integrity": "sha512-XSWeqsd3rKSsT3WBz/JKJDcZD4QYElZEa0xVdX8f9dh4h4QgXhKRLorVsVkK3uXFbC2sZKAS2Ds+YolGwD83Dg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/redis-common": "^0.38.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-kafkajs": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-kafkajs/-/instrumentation-kafkajs-0.18.0.tgz", - "integrity": "sha512-KCL/1HnZN5zkUMgPyOxfGjLjbXjpd4odDToy+7c+UsthIzVLFf99LnfIBE8YSSrYE4+uS7OwJMhvhg3tWjqMBg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.30.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-knex": { - "version": "0.53.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-knex/-/instrumentation-knex-0.53.0.tgz", - "integrity": "sha512-xngn5cH2mVXFmiT1XfQ1aHqq1m4xb5wvU6j9lSgLlihJ1bXzsO543cpDwjrZm2nMrlpddBf55w8+bfS4qDh60g==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.33.1" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-koa": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-koa/-/instrumentation-koa-0.57.0.tgz", - "integrity": "sha512-3JS8PU/D5E3q295mwloU2v7c7/m+DyCqdu62BIzWt+3u9utjxC9QS7v6WmUNuoDN3RM+Q+D1Gpj13ERo+m7CGg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.36.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0" - } - }, - "node_modules/@opentelemetry/instrumentation-lru-memoizer": { - "version": "0.53.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.53.0.tgz", - "integrity": "sha512-LDwWz5cPkWWr0HBIuZUjslyvijljTwmwiItpMTHujaULZCxcYE9eU44Qf/pbVC8TulT0IhZi+RoGvHKXvNhysw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mongodb": { - "version": "0.61.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.61.0.tgz", - "integrity": "sha512-OV3i2DSoY5M/pmLk+68xr5RvkHU8DRB3DKMzYJdwDdcxeLs62tLbkmRyqJZsYf3Ht7j11rq35pHOWLuLzXL7pQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mongoose": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.55.0.tgz", - "integrity": "sha512-5afj0HfF6aM6Nlqgu6/PPHFk8QBfIe3+zF9FGpX76jWPS0/dujoEYn82/XcLSaW5LPUDW8sni+YeK0vTBNri+w==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mysql": { - "version": "0.54.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql/-/instrumentation-mysql-0.54.0.tgz", - "integrity": "sha512-bqC1YhnwAeWmRzy1/Xf9cDqxNG2d/JDkaxnqF5N6iJKN1eVWI+vg7NfDkf52/Nggp3tl1jcC++ptC61BD6738A==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@types/mysql": "2.15.27" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mysql2": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql2/-/instrumentation-mysql2-0.55.0.tgz", - "integrity": "sha512-0cs8whQG55aIi20gnK8B7cco6OK6N+enNhW0p5284MvqJ5EPi+I1YlWsWXgzv/V2HFirEejkvKiI4Iw21OqDWg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.33.0", - "@opentelemetry/sql-common": "^0.41.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-pg": { - "version": "0.61.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.61.0.tgz", - "integrity": "sha512-UeV7KeTnRSM7ECHa3YscoklhUtTQPs6V6qYpG283AB7xpnPGCUCUfECFT9jFg6/iZOQTt3FHkB1wGTJCNZEvPw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.34.0", - "@opentelemetry/sql-common": "^0.41.2", - "@types/pg": "8.15.6", - "@types/pg-pool": "2.0.6" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-redis": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-redis/-/instrumentation-redis-0.57.0.tgz", - "integrity": "sha512-bCxTHQFXzrU3eU1LZnOZQ3s5LURxQPDlU3/upBzlWY77qOI1GZuGofazj3jtzjctMJeBEJhNwIFEgRPBX1kp/Q==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/redis-common": "^0.38.2", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-tedious": { - "version": "0.27.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-tedious/-/instrumentation-tedious-0.27.0.tgz", - "integrity": "sha512-jRtyUJNZppPBjPae4ZjIQ2eqJbcRaRfJkr0lQLHFmOU/no5A6e9s1OHLd5XZyZoBJ/ymngZitanyRRA5cniseA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@types/tedious": "^4.0.14" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-undici": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.19.0.tgz", - "integrity": "sha512-Pst/RhR61A2OoZQZkn6OLpdVpXp6qn3Y92wXa6umfJe9rV640r4bc6SWvw4pPN6DiQqPu2c8gnSSZPDtC6JlpQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.24.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.7.0" - } - }, - "node_modules/@opentelemetry/redis-common": { - "version": "0.38.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/redis-common/-/redis-common-0.38.2.tgz", - "integrity": "sha512-1BCcU93iwSRZvDAgwUxC/DV4T/406SkMfxGqu5ojc3AvNI+I9GhV7v0J1HljsczuuhcnFLYqD5VmwVXfCGHzxA==", - "license": "Apache-2.0", - "engines": { - "node": "^18.19.0 || >=20.6.0" - } - }, - "node_modules/@opentelemetry/resources": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.2.0.tgz", - "integrity": "sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/sdk-trace-base": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.2.0.tgz", - "integrity": "sha512-xWQgL0Bmctsalg6PaXExmzdedSp3gyKV8mQBwK/j9VGdCDu2fmXIb2gAehBKbkXCpJ4HPkgv3QfoJWRT4dHWbw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/resources": "2.2.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/semantic-conventions": { - "version": "1.38.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.38.0.tgz", - "integrity": "sha512-kocjix+/sSggfJhwXqClZ3i9Y/MI0fp7b+g7kCRm6psy2dsf8uApTRclwG18h8Avm7C9+fnt+O36PspJ/OzoWg==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/@opentelemetry/sql-common": { - "version": "0.41.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/sql-common/-/sql-common-0.41.2.tgz", - "integrity": "sha512-4mhWm3Z8z+i508zQJ7r6Xi7y4mmoJpdvH0fZPFRkWrdp5fq7hhZ2HhYokEOLkfqSMgPR4Z9EyB3DBkbKGOqZiQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.1.0" - } - }, - "node_modules/@prisma/instrumentation": { - "version": "6.19.0", - "resolved": "https://registry.npmjs.org/@prisma/instrumentation/-/instrumentation-6.19.0.tgz", - "integrity": "sha512-QcuYy25pkXM8BJ37wVFBO7Zh34nyRV1GOb2n3lPkkbRYfl4hWl3PTcImP41P0KrzVXfa/45p6eVCos27x3exIg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": ">=0.52.0 <1" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.8" - } - }, - "node_modules/@sentry/core": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/core/-/core-10.34.0.tgz", - "integrity": "sha512-4FFpYBMf0VFdPcsr4grDYDOR87mRu6oCfb51oQjU/Pndmty7UgYo0Bst3LEC/8v0SpytBtzXq+Wx/fkwulBesg==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@sentry/node": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/node/-/node-10.34.0.tgz", - "integrity": "sha512-bEOyH97HuVtWZYAZ5mp0NhYNc+n6QCfiKuLee2P75n2kt4cIPTGvLOSdUwwjllf795uOdKZJuM1IUN0W+YMcVg==", - "license": "MIT", - "dependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^2.2.0", - "@opentelemetry/core": "^2.2.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/instrumentation-amqplib": "0.55.0", - "@opentelemetry/instrumentation-connect": "0.52.0", - "@opentelemetry/instrumentation-dataloader": "0.26.0", - "@opentelemetry/instrumentation-express": "0.57.0", - "@opentelemetry/instrumentation-fs": "0.28.0", - "@opentelemetry/instrumentation-generic-pool": "0.52.0", - "@opentelemetry/instrumentation-graphql": "0.56.0", - "@opentelemetry/instrumentation-hapi": "0.55.0", - "@opentelemetry/instrumentation-http": "0.208.0", - "@opentelemetry/instrumentation-ioredis": "0.56.0", - "@opentelemetry/instrumentation-kafkajs": "0.18.0", - "@opentelemetry/instrumentation-knex": "0.53.0", - "@opentelemetry/instrumentation-koa": "0.57.0", - "@opentelemetry/instrumentation-lru-memoizer": "0.53.0", - "@opentelemetry/instrumentation-mongodb": "0.61.0", - "@opentelemetry/instrumentation-mongoose": "0.55.0", - "@opentelemetry/instrumentation-mysql": "0.54.0", - "@opentelemetry/instrumentation-mysql2": "0.55.0", - "@opentelemetry/instrumentation-pg": "0.61.0", - "@opentelemetry/instrumentation-redis": "0.57.0", - "@opentelemetry/instrumentation-tedious": "0.27.0", - "@opentelemetry/instrumentation-undici": "0.19.0", - "@opentelemetry/resources": "^2.2.0", - "@opentelemetry/sdk-trace-base": "^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0", - "@prisma/instrumentation": "6.19.0", - "@sentry/core": "10.34.0", - "@sentry/node-core": "10.34.0", - "@sentry/opentelemetry": "10.34.0", - "import-in-the-middle": "^2.0.1", - "minimatch": "^9.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@sentry/node-core": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/node-core/-/node-core-10.34.0.tgz", - "integrity": "sha512-FrGfC8GzD1cnZDO3zwQ4cjyoY1ZwNHvZbXSvXRYxpjhXidZhvaPurjgLRSB0xGaFgoemmOp1ufsx/w6fQOGA6Q==", - "license": "MIT", - "dependencies": { - "@apm-js-collab/tracing-hooks": "^0.3.1", - "@sentry/core": "10.34.0", - "@sentry/opentelemetry": "10.34.0", - "import-in-the-middle": "^2.0.1" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/core": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/instrumentation": ">=0.57.1 <1", - "@opentelemetry/resources": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0" - } - }, - "node_modules/@sentry/opentelemetry": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/opentelemetry/-/opentelemetry-10.34.0.tgz", - "integrity": "sha512-uKuULBOmdVu3bYdD8doMLqKgN0PP3WWtI7Shu11P9PVrhSNT4U9yM9Z6v1aFlQcbrgyg3LynZuXs8lyjt90UbA==", - "license": "MIT", - "dependencies": { - "@sentry/core": "10.34.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/core": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0" - } - }, - "node_modules/@types/connect": { - "version": "3.4.38", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", - "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/json-schema": { - "version": "7.0.15", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", - "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "license": "MIT" - }, - "node_modules/@types/mysql": { - "version": "2.15.27", - "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.27.tgz", - "integrity": "sha512-YfWiV16IY0OeBfBCk8+hXKmdTKrKlwKN1MNKAPBu5JYxLwBEZl7QzeEpGnlZb3VMGJrrGmB84gXiH+ofs/TezA==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/node": { - "version": "24.10.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", - "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", - "license": "MIT", - "dependencies": { - "undici-types": "~7.16.0" - } - }, - "node_modules/@types/node-fetch": { - "version": "2.6.13", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", - "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.4" - } - }, - "node_modules/@types/pg": { - "version": "8.15.6", - "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.15.6.tgz", - "integrity": "sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "pg-protocol": "*", - "pg-types": "^2.2.0" - } - }, - "node_modules/@types/pg-pool": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@types/pg-pool/-/pg-pool-2.0.6.tgz", - "integrity": "sha512-TaAUE5rq2VQYxab5Ts7WZhKNmuN78Q6PiFonTDdpbx8a1H0M1vhy3rhiMjl+e2iHmogyMw7jZF4FrE6eJUy5HQ==", - "license": "MIT", - "dependencies": { - "@types/pg": "*" - } - }, - "node_modules/@types/retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", - "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", - "license": "MIT" - }, - "node_modules/@types/tedious": { - "version": "4.0.14", - "resolved": "https://registry.npmjs.org/@types/tedious/-/tedious-4.0.14.tgz", - "integrity": "sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/uuid": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", - "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", - "license": "MIT" - }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "license": "MIT", - "dependencies": { - "event-target-shim": "^5.0.0" - }, - "engines": { - "node": ">=6.5" - } - }, - "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", - "license": "MIT", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-import-attributes": { - "version": "1.9.5", - "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", - "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", - "license": "MIT", - "peerDependencies": { - "acorn": "^8" - } - }, - "node_modules/agentkeepalive": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", - "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", - "license": "MIT", - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, - "node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "license": "MIT" - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/camelcase": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", - "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/chalk/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/cjs-module-lexer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.2.0.tgz", - "integrity": "sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==", - "license": "MIT" - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "license": "MIT" - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/console-table-printer": { - "version": "2.15.0", - "resolved": "https://registry.npmjs.org/console-table-printer/-/console-table-printer-2.15.0.tgz", - "integrity": "sha512-SrhBq4hYVjLCkBVOWaTzceJalvn5K1Zq5aQA6wXC/cYjI3frKWNPEMK3sZsJfNNQApvCQmgBcc13ZKmFj8qExw==", - "license": "MIT", - "dependencies": { - "simple-wcswidth": "^1.1.2" - } - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/dotenv": { - "version": "17.2.3", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", - "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/eventemitter3": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", - "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", - "license": "MIT" - }, - "node_modules/form-data": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", - "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/form-data-encoder": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", - "license": "MIT" - }, - "node_modules/formdata-node": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", - "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "license": "MIT", - "dependencies": { - "node-domexception": "1.0.0", - "web-streams-polyfill": "4.0.0-beta.3" - }, - "engines": { - "node": ">= 12.20" - } - }, - "node_modules/forwarded-parse": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/forwarded-parse/-/forwarded-parse-2.1.2.tgz", - "integrity": "sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==", - "license": "MIT" - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.0.0" - } - }, - "node_modules/import-in-the-middle": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-2.0.4.tgz", - "integrity": "sha512-Al0kMpa0BqfvDnxjxGlab9vdQ0vTDs82TBKrD59X9jReUoPAzSGBb6vGDzMUMFBGyyDF03RpLT4oxGn6BpASzQ==", - "license": "Apache-2.0", - "dependencies": { - "acorn": "^8.15.0", - "acorn-import-attributes": "^1.9.5", - "cjs-module-lexer": "^2.2.0", - "module-details-from-path": "^1.0.4" - } - }, - "node_modules/js-tiktoken": { - "version": "1.0.21", - "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", - "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", - "license": "MIT", - "dependencies": { - "base64-js": "^1.5.1" - } - }, - "node_modules/langsmith": { - "version": "0.3.79", - "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.3.79.tgz", - "integrity": "sha512-j5uiAsyy90zxlxaMuGjb7EdcL51Yx61SpKfDOI1nMPBbemGju+lf47he4e59Hp5K63CY8XWgFP42WeZ+zuIU4Q==", - "license": "MIT", - "dependencies": { - "@types/uuid": "^10.0.0", - "chalk": "^4.1.2", - "console-table-printer": "^2.12.1", - "p-queue": "^6.6.2", - "p-retry": "4", - "semver": "^7.6.3", - "uuid": "^10.0.0" - }, - "peerDependencies": { - "@opentelemetry/api": "*", - "@opentelemetry/exporter-trace-otlp-proto": "*", - "@opentelemetry/sdk-trace-base": "*", - "openai": "*" - }, - "peerDependenciesMeta": { - "@opentelemetry/api": { - "optional": true - }, - "@opentelemetry/exporter-trace-otlp-proto": { - "optional": true - }, - "@opentelemetry/sdk-trace-base": { - "optional": true - }, - "openai": { - "optional": true - } - } - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/module-details-from-path": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", - "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", - "license": "MIT" - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/mustache": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", - "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", - "license": "MIT", - "bin": { - "mustache": "bin/mustache" - } - }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/openai": { - "version": "4.104.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.104.0.tgz", - "integrity": "sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==", - "license": "Apache-2.0", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7" - }, - "bin": { - "openai": "bin/cli" - }, - "peerDependencies": { - "ws": "^8.18.0", - "zod": "^3.23.8" - }, - "peerDependenciesMeta": { - "ws": { - "optional": true - }, - "zod": { - "optional": true - } - } - }, - "node_modules/openai/node_modules/@types/node": { - "version": "18.19.130", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", - "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", - "license": "MIT", - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/openai/node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "license": "MIT" - }, - "node_modules/p-finally": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/p-queue": { - "version": "6.6.2", - "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", - "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", - "license": "MIT", - "dependencies": { - "eventemitter3": "^4.0.4", - "p-timeout": "^3.2.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-retry": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", - "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", - "license": "MIT", - "dependencies": { - "@types/retry": "0.12.0", - "retry": "^0.13.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-timeout": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", - "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", - "license": "MIT", - "dependencies": { - "p-finally": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pg-int8": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", - "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", - "license": "ISC", - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/pg-protocol": { - "version": "1.10.3", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.10.3.tgz", - "integrity": "sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==", - "license": "MIT" - }, - "node_modules/pg-types": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", - "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", - "license": "MIT", - "dependencies": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-array": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", - "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-bytea": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", - "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-date": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", - "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-interval": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", - "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", - "license": "MIT", - "dependencies": { - "xtend": "^4.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-in-the-middle": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-8.0.1.tgz", - "integrity": "sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==", - "license": "MIT", - "dependencies": { - "debug": "^4.3.5", - "module-details-from-path": "^1.0.3" - }, - "engines": { - "node": ">=9.3.0 || >=8.10.0 <9.0.0" - } - }, - "node_modules/retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/semver": { - "version": "7.7.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", - "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/simple-wcswidth": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/simple-wcswidth/-/simple-wcswidth-1.1.2.tgz", - "integrity": "sha512-j7piyCjAeTDSjzTSQ7DokZtMNwNlEAyxqSZeCS+CXH7fJ4jx3FuJ/mTW3mE+6JLs4VJBbcll0Kjn+KXI5t21Iw==", - "license": "MIT" - }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "license": "MIT" - }, - "node_modules/undici-types": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", - "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", - "license": "MIT" - }, - "node_modules/uuid": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", - "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/web-streams-polyfill": { - "version": "4.0.0-beta.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", - "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "license": "BSD-2-Clause" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "node_modules/xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "license": "MIT", - "engines": { - "node": ">=0.4" - } - }, - "node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/zod-to-json-schema": { - "version": "3.24.6", - "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.6.tgz", - "integrity": "sha512-h/z3PKvcTcTetyjl1fkj79MHNEjm+HpD6NXheWjzOekY7kV+lwDYnHw+ivHkijnCSMz1yJaWBD9vu/Fcmk+vEg==", - "license": "ISC", - "peerDependencies": { - "zod": "^3.24.1" - } - } - } -} diff --git a/sdks/js/langgraph/package.json b/sdks/js/langgraph/package.json deleted file mode 100644 index 0300f21..0000000 --- a/sdks/js/langgraph/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "@sentry-ai-sdks/langgraph", - "version": "1.0.0", - "description": "LangGraph SDK integration tests for Sentry", - "dependencies": { - "@sentry/node": "10.34.0", - "@langchain/core": "0.3.39", - "@langchain/openai": "0.4.6", - "@langchain/langgraph": "0.2.31", - "dotenv": "17.2.3" - } -} diff --git a/sdks/js/langgraph/setup.js b/sdks/js/langgraph/setup.js deleted file mode 100644 index 25c34ab..0000000 --- a/sdks/js/langgraph/setup.js +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Setup file for LangGraph SDK tests - * - * Initializes Sentry with LangChain-specific integrations (LangGraph uses LangChain). - */ - -const Sentry = require("@sentry/node"); -const { createTransport } = require("@sentry/core"); -const { config } = require("dotenv"); -const { resolve } = require("path"); -const { createMockTransport } = require("../_test-utils/mock-transport.cjs"); - -// Load environment variables -config({ quiet: true, path: resolve(__dirname, "../../../.env") }); - -// Initialize Sentry. -// NOTE: AI integrations are auto-enabled in the Node.js SDK and -// therefore do not need to be configured explicitly. -Sentry.init({ - dsn: process.env.SENTRY_DSN || "https://public@127.0.0.1/1", - tracesSampleRate: 1.0, - transport: createMockTransport(createTransport), - sendDefaultPii: true, -}); - -module.exports = { Sentry }; diff --git a/sdks/js/openai/cases/1-simple.js b/sdks/js/openai/cases/1-simple.js deleted file mode 100644 index f0570ea..0000000 --- a/sdks/js/openai/cases/1-simple.js +++ /dev/null @@ -1,38 +0,0 @@ -/** - * 1-simple: Basic Completion - * - * Tests a simple chat completion request with OpenAI SDK - * and verifies that Sentry captures the appropriate spans and AI monitoring data. - */ - -const { Sentry } = require("../setup"); -const OpenAI = require("openai"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, system, prompt } = inputs; - - const client = new OpenAI({ - apiKey: process.env.OPENAI_API_KEY, - }); - - const completion = await client.chat.completions.create({ - model, - messages: [ - { role: "system", content: system }, - { role: "user", content: prompt }, - ], - }); - - const text = completion.choices[0]?.message?.content; - - if (!text) { - throw new Error("No completion returned from OpenAI"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${text}`); - } -} - -module.exports = runTestCase("1-simple", testLogic, Sentry); diff --git a/sdks/js/openai/cases/10-binary-content-redaction.js b/sdks/js/openai/cases/10-binary-content-redaction.js deleted file mode 100644 index 536b982..0000000 --- a/sdks/js/openai/cases/10-binary-content-redaction.js +++ /dev/null @@ -1,63 +0,0 @@ -/** - * 10-binary-content-redaction: Binary Content Redaction - * - * Tests that binary image data is properly redacted in Sentry spans. - * Sends a message with a base64-encoded image and verifies that Sentry: - * - Captures the message structure - * - Redacts the binary content with "[Blob substitute]" marker - * - Does not send raw binary data to Sentry - */ - -const { Sentry } = require("../setup"); -const OpenAI = require("openai"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, image_type } = inputs; - - const client = new OpenAI({ - apiKey: process.env.OPENAI_API_KEY, - }); - - // Create a small base64-encoded image (1x1 pixel PNG) - // This is a valid 1x1 transparent PNG image - const base64Image = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; - - // Use OpenAI's vision API format (multimodal message) - // Note: Using gpt-4-vision-preview or gpt-4o for vision support - // The fixture uses gpt-5-nano but we'll let the test runner handle model override - const completion = await client.chat.completions.create({ - model, - messages: [ - { - role: "user", - content: [ - { - type: "text", - text: "What do you see in this image?", - }, - { - type: "image_url", - image_url: { - url: `data:image/${image_type};base64,${base64Image}`, - }, - }, - ], - }, - ], - }); - - const text = completion.choices[0]?.message?.content; - - if (!text) { - throw new Error("No completion returned from OpenAI"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${text}`); - console.log(` Sent message with base64-encoded ${image_type} image`); - } -} - -module.exports = runTestCase("10-binary-content-redaction", testLogic, Sentry); diff --git a/sdks/js/openai/cases/2-multi-step.js b/sdks/js/openai/cases/2-multi-step.js deleted file mode 100644 index 7c08293..0000000 --- a/sdks/js/openai/cases/2-multi-step.js +++ /dev/null @@ -1,60 +0,0 @@ -/** - * 2-multi-step: Multi-step Conversation - * - * Tests a multi-step conversation with conversation history using OpenAI SDK - * and verifies that Sentry captures all spans for both API calls. - */ - -const { Sentry } = require("../setup"); -const OpenAI = require("openai"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, system, first_prompt, second_prompt } = inputs; - - const client = new OpenAI({ - apiKey: process.env.OPENAI_API_KEY, - }); - - // First call - const firstCompletion = await client.chat.completions.create({ - model, - messages: [ - { role: "system", content: system }, - { role: "user", content: first_prompt }, - ], - }); - - const firstText = firstCompletion.choices[0]?.message?.content; - - if (!firstText) { - throw new Error("No completion returned from OpenAI (first call)"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` First response: ${firstText}`); - } - - // Second call with conversation history - const secondCompletion = await client.chat.completions.create({ - model, - messages: [ - { role: "system", content: system }, - { role: "user", content: first_prompt }, - { role: "assistant", content: firstText }, - { role: "user", content: second_prompt }, - ], - }); - - const secondText = secondCompletion.choices[0]?.message?.content; - - if (!secondText) { - throw new Error("No completion returned from OpenAI (second call)"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Second response: ${secondText}`); - } -} - -module.exports = runTestCase("2-multi-step", testLogic, Sentry); diff --git a/sdks/js/openai/cases/9-message-truncation.js b/sdks/js/openai/cases/9-message-truncation.js deleted file mode 100644 index f0bbba8..0000000 --- a/sdks/js/openai/cases/9-message-truncation.js +++ /dev/null @@ -1,53 +0,0 @@ -/** - * 9-message-truncation: Message Truncation - * - * Tests that large messages are properly truncated while preserving the original message count. - * Sends multiple large messages (~9KB each) and verifies that Sentry captures: - * - gen_ai.request.messages.original_length (the actual count of messages sent) - * - gen_ai.request.messages array (potentially truncated for telemetry) - * - The relationship: len(messages) <= original_length - */ - -const { Sentry } = require("../setup"); -const OpenAI = require("openai"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, message_size_kb, message_count } = inputs; - - const client = new OpenAI({ - apiKey: process.env.OPENAI_API_KEY, - }); - - // Generate large message content (~9KB each) - // Using approximately 9 characters per byte (rough estimate for ASCII) - const contentSize = message_size_kb * 1024; - const largeContent = "A".repeat(contentSize); - - // Build messages array with multiple large messages - const messages = []; - for (let i = 0; i < message_count; i++) { - messages.push({ - role: "user", - content: `Message ${i + 1}: ${largeContent}`, - }); - } - - const completion = await client.chat.completions.create({ - model, - messages, - }); - - const text = completion.choices[0]?.message?.content; - - if (!text) { - throw new Error("No completion returned from OpenAI"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${text}`); - console.log(` Sent ${messages.length} messages with ~${message_size_kb}KB each`); - } -} - -module.exports = runTestCase("9-message-truncation", testLogic, Sentry); diff --git a/sdks/js/openai/config.json b/sdks/js/openai/config.json deleted file mode 100644 index 28bbbb9..0000000 --- a/sdks/js/openai/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "sdk_name": "openai", - "framework_type": "low-level", - "overrides": {} -} diff --git a/sdks/js/openai/package-lock.json b/sdks/js/openai/package-lock.json deleted file mode 100644 index 1ac34d8..0000000 --- a/sdks/js/openai/package-lock.json +++ /dev/null @@ -1,957 +0,0 @@ -{ - "name": "@sentry-ai-sdks/openai", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@sentry-ai-sdks/openai", - "version": "1.0.0", - "dependencies": { - "@sentry/node": "10.34.0", - "dotenv": "17.2.3", - "openai": "6.8.1" - } - }, - "node_modules/@apm-js-collab/code-transformer": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/@apm-js-collab/code-transformer/-/code-transformer-0.8.2.tgz", - "integrity": "sha512-YRjJjNq5KFSjDUoqu5pFUWrrsvGOxl6c3bu+uMFc9HNNptZ2rNU/TI2nLw4jnhQNtka972Ee2m3uqbvDQtPeCA==", - "license": "Apache-2.0" - }, - "node_modules/@apm-js-collab/tracing-hooks": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@apm-js-collab/tracing-hooks/-/tracing-hooks-0.3.1.tgz", - "integrity": "sha512-Vu1CbmPURlN5fTboVuKMoJjbO5qcq9fA5YXpskx3dXe/zTBvjODFoerw+69rVBlRLrJpwPqSDqEuJDEKIrTldw==", - "license": "Apache-2.0", - "dependencies": { - "@apm-js-collab/code-transformer": "^0.8.0", - "debug": "^4.4.1", - "module-details-from-path": "^1.0.4" - } - }, - "node_modules/@opentelemetry/api": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", - "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", - "license": "Apache-2.0", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/api-logs": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.208.0.tgz", - "integrity": "sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api": "^1.3.0" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/context-async-hooks": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.4.0.tgz", - "integrity": "sha512-jn0phJ+hU7ZuvaoZE/8/Euw3gvHJrn2yi+kXrymwObEPVPjtwCmkvXDRQCWli+fCTTF/aSOtXaLr7CLIvv3LQg==", - "license": "Apache-2.0", - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/core": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.2.0.tgz", - "integrity": "sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/instrumentation": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.208.0.tgz", - "integrity": "sha512-Eju0L4qWcQS+oXxi6pgh7zvE2byogAkcsVv0OjHF/97iOz1N/aKE6etSGowYkie+YA1uo6DNwdSxaaNnLvcRlA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api-logs": "0.208.0", - "import-in-the-middle": "^2.0.0", - "require-in-the-middle": "^8.0.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-amqplib": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-amqplib/-/instrumentation-amqplib-0.55.0.tgz", - "integrity": "sha512-5ULoU8p+tWcQw5PDYZn8rySptGSLZHNX/7srqo2TioPnAAcvTy6sQFQXsNPrAnyRRtYGMetXVyZUy5OaX1+IfA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-connect": { - "version": "0.52.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-connect/-/instrumentation-connect-0.52.0.tgz", - "integrity": "sha512-GXPxfNB5szMbV3I9b7kNWSmQBoBzw7MT0ui6iU/p+NIzVx3a06Ri2cdQO7tG9EKb4aKSLmfX9Cw5cKxXqX6Ohg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0", - "@types/connect": "3.4.38" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-dataloader": { - "version": "0.26.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-dataloader/-/instrumentation-dataloader-0.26.0.tgz", - "integrity": "sha512-P2BgnFfTOarZ5OKPmYfbXfDFjQ4P9WkQ1Jji7yH5/WwB6Wm/knynAoA1rxbjWcDlYupFkyT0M1j6XLzDzy0aCA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-express": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-express/-/instrumentation-express-0.57.0.tgz", - "integrity": "sha512-HAdx/o58+8tSR5iW+ru4PHnEejyKrAy9fYFhlEI81o10nYxrGahnMAHWiSjhDC7UQSY3I4gjcPgSKQz4rm/asg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-fs": { - "version": "0.28.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.28.0.tgz", - "integrity": "sha512-FFvg8fq53RRXVBRHZViP+EMxMR03tqzEGpuq55lHNbVPyFklSVfQBN50syPhK5UYYwaStx0eyCtHtbRreusc5g==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-generic-pool": { - "version": "0.52.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-generic-pool/-/instrumentation-generic-pool-0.52.0.tgz", - "integrity": "sha512-ISkNcv5CM2IwvsMVL31Tl61/p2Zm2I2NAsYq5SSBgOsOndT0TjnptjufYVScCnD5ZLD1tpl4T3GEYULLYOdIdQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-graphql": { - "version": "0.56.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-graphql/-/instrumentation-graphql-0.56.0.tgz", - "integrity": "sha512-IPvNk8AFoVzTAM0Z399t34VDmGDgwT6rIqCUug8P9oAGerl2/PEIYMPOl/rerPGu+q8gSWdmbFSjgg7PDVRd3Q==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-hapi": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-hapi/-/instrumentation-hapi-0.55.0.tgz", - "integrity": "sha512-prqAkRf9e4eEpy4G3UcR32prKE8NLNlA90TdEU1UsghOTg0jUvs40Jz8LQWFEs5NbLbXHYGzB4CYVkCI8eWEVQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-http": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.208.0.tgz", - "integrity": "sha512-rhmK46DRWEbQQB77RxmVXGyjs6783crXCnFjYQj+4tDH/Kpv9Rbg3h2kaNyp5Vz2emF1f9HOQQvZoHzwMWOFZQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/instrumentation": "0.208.0", - "@opentelemetry/semantic-conventions": "^1.29.0", - "forwarded-parse": "2.1.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-ioredis": { - "version": "0.56.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.56.0.tgz", - "integrity": "sha512-XSWeqsd3rKSsT3WBz/JKJDcZD4QYElZEa0xVdX8f9dh4h4QgXhKRLorVsVkK3uXFbC2sZKAS2Ds+YolGwD83Dg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/redis-common": "^0.38.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-kafkajs": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-kafkajs/-/instrumentation-kafkajs-0.18.0.tgz", - "integrity": "sha512-KCL/1HnZN5zkUMgPyOxfGjLjbXjpd4odDToy+7c+UsthIzVLFf99LnfIBE8YSSrYE4+uS7OwJMhvhg3tWjqMBg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.30.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-knex": { - "version": "0.53.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-knex/-/instrumentation-knex-0.53.0.tgz", - "integrity": "sha512-xngn5cH2mVXFmiT1XfQ1aHqq1m4xb5wvU6j9lSgLlihJ1bXzsO543cpDwjrZm2nMrlpddBf55w8+bfS4qDh60g==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.33.1" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-koa": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-koa/-/instrumentation-koa-0.57.0.tgz", - "integrity": "sha512-3JS8PU/D5E3q295mwloU2v7c7/m+DyCqdu62BIzWt+3u9utjxC9QS7v6WmUNuoDN3RM+Q+D1Gpj13ERo+m7CGg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.36.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0" - } - }, - "node_modules/@opentelemetry/instrumentation-lru-memoizer": { - "version": "0.53.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.53.0.tgz", - "integrity": "sha512-LDwWz5cPkWWr0HBIuZUjslyvijljTwmwiItpMTHujaULZCxcYE9eU44Qf/pbVC8TulT0IhZi+RoGvHKXvNhysw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mongodb": { - "version": "0.61.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.61.0.tgz", - "integrity": "sha512-OV3i2DSoY5M/pmLk+68xr5RvkHU8DRB3DKMzYJdwDdcxeLs62tLbkmRyqJZsYf3Ht7j11rq35pHOWLuLzXL7pQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mongoose": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.55.0.tgz", - "integrity": "sha512-5afj0HfF6aM6Nlqgu6/PPHFk8QBfIe3+zF9FGpX76jWPS0/dujoEYn82/XcLSaW5LPUDW8sni+YeK0vTBNri+w==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mysql": { - "version": "0.54.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql/-/instrumentation-mysql-0.54.0.tgz", - "integrity": "sha512-bqC1YhnwAeWmRzy1/Xf9cDqxNG2d/JDkaxnqF5N6iJKN1eVWI+vg7NfDkf52/Nggp3tl1jcC++ptC61BD6738A==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@types/mysql": "2.15.27" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mysql2": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql2/-/instrumentation-mysql2-0.55.0.tgz", - "integrity": "sha512-0cs8whQG55aIi20gnK8B7cco6OK6N+enNhW0p5284MvqJ5EPi+I1YlWsWXgzv/V2HFirEejkvKiI4Iw21OqDWg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.33.0", - "@opentelemetry/sql-common": "^0.41.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-pg": { - "version": "0.61.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.61.0.tgz", - "integrity": "sha512-UeV7KeTnRSM7ECHa3YscoklhUtTQPs6V6qYpG283AB7xpnPGCUCUfECFT9jFg6/iZOQTt3FHkB1wGTJCNZEvPw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.34.0", - "@opentelemetry/sql-common": "^0.41.2", - "@types/pg": "8.15.6", - "@types/pg-pool": "2.0.6" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-redis": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-redis/-/instrumentation-redis-0.57.0.tgz", - "integrity": "sha512-bCxTHQFXzrU3eU1LZnOZQ3s5LURxQPDlU3/upBzlWY77qOI1GZuGofazj3jtzjctMJeBEJhNwIFEgRPBX1kp/Q==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/redis-common": "^0.38.2", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-tedious": { - "version": "0.27.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-tedious/-/instrumentation-tedious-0.27.0.tgz", - "integrity": "sha512-jRtyUJNZppPBjPae4ZjIQ2eqJbcRaRfJkr0lQLHFmOU/no5A6e9s1OHLd5XZyZoBJ/ymngZitanyRRA5cniseA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@types/tedious": "^4.0.14" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-undici": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.19.0.tgz", - "integrity": "sha512-Pst/RhR61A2OoZQZkn6OLpdVpXp6qn3Y92wXa6umfJe9rV640r4bc6SWvw4pPN6DiQqPu2c8gnSSZPDtC6JlpQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.24.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.7.0" - } - }, - "node_modules/@opentelemetry/redis-common": { - "version": "0.38.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/redis-common/-/redis-common-0.38.2.tgz", - "integrity": "sha512-1BCcU93iwSRZvDAgwUxC/DV4T/406SkMfxGqu5ojc3AvNI+I9GhV7v0J1HljsczuuhcnFLYqD5VmwVXfCGHzxA==", - "license": "Apache-2.0", - "engines": { - "node": "^18.19.0 || >=20.6.0" - } - }, - "node_modules/@opentelemetry/resources": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.4.0.tgz", - "integrity": "sha512-RWvGLj2lMDZd7M/5tjkI/2VHMpXebLgPKvBUd9LRasEWR2xAynDwEYZuLvY9P2NGG73HF07jbbgWX2C9oavcQg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.4.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/resources/node_modules/@opentelemetry/core": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.4.0.tgz", - "integrity": "sha512-KtcyFHssTn5ZgDu6SXmUznS80OFs/wN7y6MyFRRcKU6TOw8hNcGxKvt8hsdaLJfhzUszNSjURetq5Qpkad14Gw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/sdk-trace-base": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.4.0.tgz", - "integrity": "sha512-WH0xXkz/OHORDLKqaxcUZS0X+t1s7gGlumr2ebiEgNZQl2b0upK2cdoD0tatf7l8iP74woGJ/Kmxe82jdvcWRw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.4.0", - "@opentelemetry/resources": "2.4.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/sdk-trace-base/node_modules/@opentelemetry/core": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.4.0.tgz", - "integrity": "sha512-KtcyFHssTn5ZgDu6SXmUznS80OFs/wN7y6MyFRRcKU6TOw8hNcGxKvt8hsdaLJfhzUszNSjURetq5Qpkad14Gw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/semantic-conventions": { - "version": "1.38.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.38.0.tgz", - "integrity": "sha512-kocjix+/sSggfJhwXqClZ3i9Y/MI0fp7b+g7kCRm6psy2dsf8uApTRclwG18h8Avm7C9+fnt+O36PspJ/OzoWg==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/@opentelemetry/sql-common": { - "version": "0.41.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/sql-common/-/sql-common-0.41.2.tgz", - "integrity": "sha512-4mhWm3Z8z+i508zQJ7r6Xi7y4mmoJpdvH0fZPFRkWrdp5fq7hhZ2HhYokEOLkfqSMgPR4Z9EyB3DBkbKGOqZiQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.1.0" - } - }, - "node_modules/@prisma/instrumentation": { - "version": "6.19.0", - "resolved": "https://registry.npmjs.org/@prisma/instrumentation/-/instrumentation-6.19.0.tgz", - "integrity": "sha512-QcuYy25pkXM8BJ37wVFBO7Zh34nyRV1GOb2n3lPkkbRYfl4hWl3PTcImP41P0KrzVXfa/45p6eVCos27x3exIg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": ">=0.52.0 <1" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.8" - } - }, - "node_modules/@sentry/core": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/core/-/core-10.34.0.tgz", - "integrity": "sha512-4FFpYBMf0VFdPcsr4grDYDOR87mRu6oCfb51oQjU/Pndmty7UgYo0Bst3LEC/8v0SpytBtzXq+Wx/fkwulBesg==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@sentry/node": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/node/-/node-10.34.0.tgz", - "integrity": "sha512-bEOyH97HuVtWZYAZ5mp0NhYNc+n6QCfiKuLee2P75n2kt4cIPTGvLOSdUwwjllf795uOdKZJuM1IUN0W+YMcVg==", - "license": "MIT", - "dependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^2.2.0", - "@opentelemetry/core": "^2.2.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/instrumentation-amqplib": "0.55.0", - "@opentelemetry/instrumentation-connect": "0.52.0", - "@opentelemetry/instrumentation-dataloader": "0.26.0", - "@opentelemetry/instrumentation-express": "0.57.0", - "@opentelemetry/instrumentation-fs": "0.28.0", - "@opentelemetry/instrumentation-generic-pool": "0.52.0", - "@opentelemetry/instrumentation-graphql": "0.56.0", - "@opentelemetry/instrumentation-hapi": "0.55.0", - "@opentelemetry/instrumentation-http": "0.208.0", - "@opentelemetry/instrumentation-ioredis": "0.56.0", - "@opentelemetry/instrumentation-kafkajs": "0.18.0", - "@opentelemetry/instrumentation-knex": "0.53.0", - "@opentelemetry/instrumentation-koa": "0.57.0", - "@opentelemetry/instrumentation-lru-memoizer": "0.53.0", - "@opentelemetry/instrumentation-mongodb": "0.61.0", - "@opentelemetry/instrumentation-mongoose": "0.55.0", - "@opentelemetry/instrumentation-mysql": "0.54.0", - "@opentelemetry/instrumentation-mysql2": "0.55.0", - "@opentelemetry/instrumentation-pg": "0.61.0", - "@opentelemetry/instrumentation-redis": "0.57.0", - "@opentelemetry/instrumentation-tedious": "0.27.0", - "@opentelemetry/instrumentation-undici": "0.19.0", - "@opentelemetry/resources": "^2.2.0", - "@opentelemetry/sdk-trace-base": "^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0", - "@prisma/instrumentation": "6.19.0", - "@sentry/core": "10.34.0", - "@sentry/node-core": "10.34.0", - "@sentry/opentelemetry": "10.34.0", - "import-in-the-middle": "^2.0.1", - "minimatch": "^9.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@sentry/node-core": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/node-core/-/node-core-10.34.0.tgz", - "integrity": "sha512-FrGfC8GzD1cnZDO3zwQ4cjyoY1ZwNHvZbXSvXRYxpjhXidZhvaPurjgLRSB0xGaFgoemmOp1ufsx/w6fQOGA6Q==", - "license": "MIT", - "dependencies": { - "@apm-js-collab/tracing-hooks": "^0.3.1", - "@sentry/core": "10.34.0", - "@sentry/opentelemetry": "10.34.0", - "import-in-the-middle": "^2.0.1" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/core": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/instrumentation": ">=0.57.1 <1", - "@opentelemetry/resources": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0" - } - }, - "node_modules/@sentry/opentelemetry": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/opentelemetry/-/opentelemetry-10.34.0.tgz", - "integrity": "sha512-uKuULBOmdVu3bYdD8doMLqKgN0PP3WWtI7Shu11P9PVrhSNT4U9yM9Z6v1aFlQcbrgyg3LynZuXs8lyjt90UbA==", - "license": "MIT", - "dependencies": { - "@sentry/core": "10.34.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/core": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0" - } - }, - "node_modules/@types/connect": { - "version": "3.4.38", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", - "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/mysql": { - "version": "2.15.27", - "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.27.tgz", - "integrity": "sha512-YfWiV16IY0OeBfBCk8+hXKmdTKrKlwKN1MNKAPBu5JYxLwBEZl7QzeEpGnlZb3VMGJrrGmB84gXiH+ofs/TezA==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/node": { - "version": "24.10.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", - "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", - "license": "MIT", - "dependencies": { - "undici-types": "~7.16.0" - } - }, - "node_modules/@types/pg": { - "version": "8.15.6", - "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.15.6.tgz", - "integrity": "sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "pg-protocol": "*", - "pg-types": "^2.2.0" - } - }, - "node_modules/@types/pg-pool": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@types/pg-pool/-/pg-pool-2.0.6.tgz", - "integrity": "sha512-TaAUE5rq2VQYxab5Ts7WZhKNmuN78Q6PiFonTDdpbx8a1H0M1vhy3rhiMjl+e2iHmogyMw7jZF4FrE6eJUy5HQ==", - "license": "MIT", - "dependencies": { - "@types/pg": "*" - } - }, - "node_modules/@types/tedious": { - "version": "4.0.14", - "resolved": "https://registry.npmjs.org/@types/tedious/-/tedious-4.0.14.tgz", - "integrity": "sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", - "license": "MIT", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-import-attributes": { - "version": "1.9.5", - "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", - "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", - "license": "MIT", - "peerDependencies": { - "acorn": "^8" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "license": "MIT" - }, - "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/cjs-module-lexer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.2.0.tgz", - "integrity": "sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==", - "license": "MIT" - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/dotenv": { - "version": "17.2.3", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", - "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/forwarded-parse": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/forwarded-parse/-/forwarded-parse-2.1.2.tgz", - "integrity": "sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==", - "license": "MIT" - }, - "node_modules/import-in-the-middle": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-2.0.4.tgz", - "integrity": "sha512-Al0kMpa0BqfvDnxjxGlab9vdQ0vTDs82TBKrD59X9jReUoPAzSGBb6vGDzMUMFBGyyDF03RpLT4oxGn6BpASzQ==", - "license": "Apache-2.0", - "dependencies": { - "acorn": "^8.15.0", - "acorn-import-attributes": "^1.9.5", - "cjs-module-lexer": "^2.2.0", - "module-details-from-path": "^1.0.4" - } - }, - "node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/module-details-from-path": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", - "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", - "license": "MIT" - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/openai": { - "version": "6.8.1", - "resolved": "https://registry.npmjs.org/openai/-/openai-6.8.1.tgz", - "integrity": "sha512-ACifslrVgf+maMz9vqwMP4+v9qvx5Yzssydizks8n+YUJ6YwUoxj51sKRQ8HYMfR6wgKLSIlaI108ZwCk+8yig==", - "license": "Apache-2.0", - "bin": { - "openai": "bin/cli" - }, - "peerDependencies": { - "ws": "^8.18.0", - "zod": "^3.25 || ^4.0" - }, - "peerDependenciesMeta": { - "ws": { - "optional": true - }, - "zod": { - "optional": true - } - } - }, - "node_modules/pg-int8": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", - "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", - "license": "ISC", - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/pg-protocol": { - "version": "1.10.3", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.10.3.tgz", - "integrity": "sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==", - "license": "MIT" - }, - "node_modules/pg-types": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", - "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", - "license": "MIT", - "dependencies": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-array": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", - "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-bytea": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", - "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-date": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", - "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-interval": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", - "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", - "license": "MIT", - "dependencies": { - "xtend": "^4.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-in-the-middle": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-8.0.1.tgz", - "integrity": "sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==", - "license": "MIT", - "dependencies": { - "debug": "^4.3.5", - "module-details-from-path": "^1.0.3" - }, - "engines": { - "node": ">=9.3.0 || >=8.10.0 <9.0.0" - } - }, - "node_modules/undici-types": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", - "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", - "license": "MIT" - }, - "node_modules/xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "license": "MIT", - "engines": { - "node": ">=0.4" - } - } - } -} diff --git a/sdks/js/openai/package.json b/sdks/js/openai/package.json deleted file mode 100644 index 49cf808..0000000 --- a/sdks/js/openai/package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "@sentry-ai-sdks/openai", - "version": "1.0.0", - "description": "OpenAI SDK integration tests for Sentry", - "dependencies": { - "@sentry/node": "10.34.0", - "openai": "6.8.1", - "dotenv": "17.2.3" - } -} diff --git a/sdks/js/openai/setup.js b/sdks/js/openai/setup.js deleted file mode 100644 index b8a1391..0000000 --- a/sdks/js/openai/setup.js +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Setup file for OpenAI SDK tests - * - * Initializes Sentry with OpenAI-specific integrations. - */ - -const Sentry = require("@sentry/node"); -const { createTransport } = require("@sentry/core"); -const { config } = require("dotenv"); -const { resolve } = require("path"); -const { createMockTransport } = require("../_test-utils/mock-transport.cjs"); - -// Load environment variables -config({ quiet: true, path: resolve(__dirname, "../../../.env") }); - -// Initialize Sentry. -// NOTE: AI integrations are auto-enabled in the Node.js SDK and -// therefore do not need to be configured explicitly. -Sentry.init({ - dsn: process.env.SENTRY_DSN || "https://public@127.0.0.1/1", - tracesSampleRate: 1.0, - transport: createMockTransport(createTransport), - sendDefaultPii: true, -}); - -module.exports = { Sentry }; diff --git a/sdks/js/vercel/cases/1-simple.js b/sdks/js/vercel/cases/1-simple.js deleted file mode 100644 index 670922e..0000000 --- a/sdks/js/vercel/cases/1-simple.js +++ /dev/null @@ -1,31 +0,0 @@ -/** - * 1-simple: Basic Completion - * - * Tests a simple chat completion request with Vercel AI SDK - * and verifies that Sentry captures the appropriate spans and AI monitoring data. - */ - -const { Sentry } = require("../setup"); -const { generateText } = require("ai"); -const { openai } = require("@ai-sdk/openai"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, system, prompt } = inputs; - - const { text } = await generateText({ - model: openai(model), - system, - prompt, - }); - - if (!text) { - throw new Error("No completion returned from OpenAI"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${text}`); - } -} - -module.exports = runTestCase("1-simple", testLogic, Sentry); diff --git a/sdks/js/vercel/cases/10-binary-content-redaction.js b/sdks/js/vercel/cases/10-binary-content-redaction.js deleted file mode 100644 index 3b83f52..0000000 --- a/sdks/js/vercel/cases/10-binary-content-redaction.js +++ /dev/null @@ -1,54 +0,0 @@ -/** - * 10-binary-content-redaction: Binary Content Redaction - * - * Tests that when binary data (such as images) is sent to an LLM, Sentry correctly - * redacts the binary content in the captured span data and replaces it with a substitute marker. - */ - -const { Sentry } = require("../setup"); -const { generateText } = require("ai"); -const { openai } = require("@ai-sdk/openai"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, image_type } = inputs; - - // Create a small binary image (1x1 pixel PNG) - // This is a valid 1x1 transparent PNG in base64 - const smallPngBase64 = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; - - // Convert to Buffer for binary representation - const imageBuffer = Buffer.from(smallPngBase64, "base64"); - - // Send a multimodal message with image content - const { text } = await generateText({ - model: openai(model), - messages: [ - { - role: "user", - content: [ - { - type: "text", - text: "Describe this image briefly.", - }, - { - type: "image", - image: imageBuffer, - }, - ], - }, - ], - }); - - if (!text) { - throw new Error("No completion returned from OpenAI"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${text}`); - console.log(` Sent image of type: ${image_type}`); - } -} - -module.exports = runTestCase("10-binary-content-redaction", testLogic, Sentry); diff --git a/sdks/js/vercel/cases/2-multi-step.js b/sdks/js/vercel/cases/2-multi-step.js deleted file mode 100644 index 1d871c3..0000000 --- a/sdks/js/vercel/cases/2-multi-step.js +++ /dev/null @@ -1,55 +0,0 @@ -/** - * 2-multi-step: Multi-step Conversation - * - * Tests a multi-step conversation with conversation history using Vercel AI SDK - * and verifies that Sentry captures all spans for both API calls. - */ - -const { Sentry } = require("../setup"); -const { generateText } = require("ai"); -const { openai } = require("@ai-sdk/openai"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, system, first_prompt, second_prompt } = inputs; - - // First call - const firstResult = await generateText({ - model: openai(model), - system, - prompt: first_prompt, - }); - - const firstText = firstResult.text; - - if (!firstText) { - throw new Error("No completion returned from Vercel AI (first call)"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` First response: ${firstText}`); - } - - // Second call with conversation history - const secondResult = await generateText({ - model: openai(model), - system, - messages: [ - { role: "user", content: first_prompt }, - { role: "assistant", content: firstText }, - { role: "user", content: second_prompt }, - ], - }); - - const secondText = secondResult.text; - - if (!secondText) { - throw new Error("No completion returned from Vercel AI (second call)"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Second response: ${secondText}`); - } -} - -module.exports = runTestCase("2-multi-step", testLogic, Sentry); diff --git a/sdks/js/vercel/cases/9-message-truncation.js b/sdks/js/vercel/cases/9-message-truncation.js deleted file mode 100644 index 0f2bfde..0000000 --- a/sdks/js/vercel/cases/9-message-truncation.js +++ /dev/null @@ -1,51 +0,0 @@ -/** - * 9-message-truncation: Message Truncation - * - * Tests that when large messages are sent to an LLM, Sentry correctly tracks - * the original message count vs. the potentially truncated message count in the captured span data. - */ - -const { Sentry } = require("../setup"); -const { generateText } = require("ai"); -const { openai } = require("@ai-sdk/openai"); -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); - -async function testLogic(inputs) { - const { model, message_size_kb, message_count } = inputs; - - // Generate large content (~9KB each message) - const largeContent = "A".repeat(message_size_kb * 1024); - - // Build a conversation with multiple large messages - const messages = []; - - for (let i = 0; i < message_count; i++) { - // Alternate between user and assistant messages to simulate a conversation - messages.push({ - role: i % 2 === 0 ? "user" : "assistant", - content: `Message ${i + 1}: ${largeContent}`, - }); - } - - // Add final user message to prompt the model - messages.push({ - role: "user", - content: "Please provide a brief summary.", - }); - - const { text } = await generateText({ - model: openai(model), - messages, - }); - - if (!text) { - throw new Error("No completion returned from OpenAI"); - } - - if (process.env.SENTRY_AI_TEST_VERBOSE === "true") { - console.log(` Response: ${text}`); - console.log(` Sent ${messages.length} messages with large content`); - } -} - -module.exports = runTestCase("9-message-truncation", testLogic, Sentry); diff --git a/sdks/js/vercel/config.json b/sdks/js/vercel/config.json deleted file mode 100644 index b665da0..0000000 --- a/sdks/js/vercel/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "sdk_name": "vercel", - "framework_type": "agentic", - "overrides": {} -} diff --git a/sdks/js/vercel/package-lock.json b/sdks/js/vercel/package-lock.json deleted file mode 100644 index 2bb91f2..0000000 --- a/sdks/js/vercel/package-lock.json +++ /dev/null @@ -1,1057 +0,0 @@ -{ - "name": "@sentry-ai-sdks/vercel", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@sentry-ai-sdks/vercel", - "version": "1.0.0", - "dependencies": { - "@ai-sdk/openai": "2.0.62", - "@sentry/node": "10.34.0", - "ai": "5.0.87", - "dotenv": "17.2.3" - } - }, - "node_modules/@ai-sdk/gateway": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-2.0.6.tgz", - "integrity": "sha512-FmhR6Tle09I/RUda8WSPpJ57mjPWzhiVVlB50D+k+Qf/PBW0CBtnbAUxlNSR5v+NIZNLTK3C56lhb23ntEdxhQ==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "2.0.0", - "@ai-sdk/provider-utils": "3.0.16", - "@vercel/oidc": "3.0.3" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.25.76 || ^4.1.8" - } - }, - "node_modules/@ai-sdk/openai": { - "version": "2.0.62", - "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-2.0.62.tgz", - "integrity": "sha512-ZHUhUV6yyBBb0bCbuqAkML7nYIOWyXZYbZQ59mlr1TpIJzSHjQzF4BndZHIIieOMm4ZrpZw15Cn78BTyaIAUwQ==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "2.0.0", - "@ai-sdk/provider-utils": "3.0.16" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.25.76 || ^4.1.8" - } - }, - "node_modules/@ai-sdk/provider": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", - "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", - "license": "Apache-2.0", - "dependencies": { - "json-schema": "^0.4.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@ai-sdk/provider-utils": { - "version": "3.0.16", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-3.0.16.tgz", - "integrity": "sha512-lsWQY9aDXHitw7C1QRYIbVGmgwyT98TF3MfM8alNIXKpdJdi+W782Rzd9f1RyOfgRmZ08gJ2EYNDhWNK7RqpEA==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/provider": "2.0.0", - "@standard-schema/spec": "^1.0.0", - "eventsource-parser": "^3.0.6" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.25.76 || ^4.1.8" - } - }, - "node_modules/@apm-js-collab/code-transformer": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/@apm-js-collab/code-transformer/-/code-transformer-0.8.2.tgz", - "integrity": "sha512-YRjJjNq5KFSjDUoqu5pFUWrrsvGOxl6c3bu+uMFc9HNNptZ2rNU/TI2nLw4jnhQNtka972Ee2m3uqbvDQtPeCA==", - "license": "Apache-2.0" - }, - "node_modules/@apm-js-collab/tracing-hooks": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@apm-js-collab/tracing-hooks/-/tracing-hooks-0.3.1.tgz", - "integrity": "sha512-Vu1CbmPURlN5fTboVuKMoJjbO5qcq9fA5YXpskx3dXe/zTBvjODFoerw+69rVBlRLrJpwPqSDqEuJDEKIrTldw==", - "license": "Apache-2.0", - "dependencies": { - "@apm-js-collab/code-transformer": "^0.8.0", - "debug": "^4.4.1", - "module-details-from-path": "^1.0.4" - } - }, - "node_modules/@opentelemetry/api": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", - "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", - "license": "Apache-2.0", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/api-logs": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.208.0.tgz", - "integrity": "sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api": "^1.3.0" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/context-async-hooks": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.4.0.tgz", - "integrity": "sha512-jn0phJ+hU7ZuvaoZE/8/Euw3gvHJrn2yi+kXrymwObEPVPjtwCmkvXDRQCWli+fCTTF/aSOtXaLr7CLIvv3LQg==", - "license": "Apache-2.0", - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/core": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.2.0.tgz", - "integrity": "sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/instrumentation": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.208.0.tgz", - "integrity": "sha512-Eju0L4qWcQS+oXxi6pgh7zvE2byogAkcsVv0OjHF/97iOz1N/aKE6etSGowYkie+YA1uo6DNwdSxaaNnLvcRlA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api-logs": "0.208.0", - "import-in-the-middle": "^2.0.0", - "require-in-the-middle": "^8.0.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-amqplib": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-amqplib/-/instrumentation-amqplib-0.55.0.tgz", - "integrity": "sha512-5ULoU8p+tWcQw5PDYZn8rySptGSLZHNX/7srqo2TioPnAAcvTy6sQFQXsNPrAnyRRtYGMetXVyZUy5OaX1+IfA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-connect": { - "version": "0.52.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-connect/-/instrumentation-connect-0.52.0.tgz", - "integrity": "sha512-GXPxfNB5szMbV3I9b7kNWSmQBoBzw7MT0ui6iU/p+NIzVx3a06Ri2cdQO7tG9EKb4aKSLmfX9Cw5cKxXqX6Ohg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0", - "@types/connect": "3.4.38" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-dataloader": { - "version": "0.26.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-dataloader/-/instrumentation-dataloader-0.26.0.tgz", - "integrity": "sha512-P2BgnFfTOarZ5OKPmYfbXfDFjQ4P9WkQ1Jji7yH5/WwB6Wm/knynAoA1rxbjWcDlYupFkyT0M1j6XLzDzy0aCA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-express": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-express/-/instrumentation-express-0.57.0.tgz", - "integrity": "sha512-HAdx/o58+8tSR5iW+ru4PHnEejyKrAy9fYFhlEI81o10nYxrGahnMAHWiSjhDC7UQSY3I4gjcPgSKQz4rm/asg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-fs": { - "version": "0.28.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.28.0.tgz", - "integrity": "sha512-FFvg8fq53RRXVBRHZViP+EMxMR03tqzEGpuq55lHNbVPyFklSVfQBN50syPhK5UYYwaStx0eyCtHtbRreusc5g==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-generic-pool": { - "version": "0.52.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-generic-pool/-/instrumentation-generic-pool-0.52.0.tgz", - "integrity": "sha512-ISkNcv5CM2IwvsMVL31Tl61/p2Zm2I2NAsYq5SSBgOsOndT0TjnptjufYVScCnD5ZLD1tpl4T3GEYULLYOdIdQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-graphql": { - "version": "0.56.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-graphql/-/instrumentation-graphql-0.56.0.tgz", - "integrity": "sha512-IPvNk8AFoVzTAM0Z399t34VDmGDgwT6rIqCUug8P9oAGerl2/PEIYMPOl/rerPGu+q8gSWdmbFSjgg7PDVRd3Q==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-hapi": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-hapi/-/instrumentation-hapi-0.55.0.tgz", - "integrity": "sha512-prqAkRf9e4eEpy4G3UcR32prKE8NLNlA90TdEU1UsghOTg0jUvs40Jz8LQWFEs5NbLbXHYGzB4CYVkCI8eWEVQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-http": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.208.0.tgz", - "integrity": "sha512-rhmK46DRWEbQQB77RxmVXGyjs6783crXCnFjYQj+4tDH/Kpv9Rbg3h2kaNyp5Vz2emF1f9HOQQvZoHzwMWOFZQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/instrumentation": "0.208.0", - "@opentelemetry/semantic-conventions": "^1.29.0", - "forwarded-parse": "2.1.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-ioredis": { - "version": "0.56.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.56.0.tgz", - "integrity": "sha512-XSWeqsd3rKSsT3WBz/JKJDcZD4QYElZEa0xVdX8f9dh4h4QgXhKRLorVsVkK3uXFbC2sZKAS2Ds+YolGwD83Dg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/redis-common": "^0.38.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-kafkajs": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-kafkajs/-/instrumentation-kafkajs-0.18.0.tgz", - "integrity": "sha512-KCL/1HnZN5zkUMgPyOxfGjLjbXjpd4odDToy+7c+UsthIzVLFf99LnfIBE8YSSrYE4+uS7OwJMhvhg3tWjqMBg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.30.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-knex": { - "version": "0.53.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-knex/-/instrumentation-knex-0.53.0.tgz", - "integrity": "sha512-xngn5cH2mVXFmiT1XfQ1aHqq1m4xb5wvU6j9lSgLlihJ1bXzsO543cpDwjrZm2nMrlpddBf55w8+bfS4qDh60g==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.33.1" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-koa": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-koa/-/instrumentation-koa-0.57.0.tgz", - "integrity": "sha512-3JS8PU/D5E3q295mwloU2v7c7/m+DyCqdu62BIzWt+3u9utjxC9QS7v6WmUNuoDN3RM+Q+D1Gpj13ERo+m7CGg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.36.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0" - } - }, - "node_modules/@opentelemetry/instrumentation-lru-memoizer": { - "version": "0.53.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.53.0.tgz", - "integrity": "sha512-LDwWz5cPkWWr0HBIuZUjslyvijljTwmwiItpMTHujaULZCxcYE9eU44Qf/pbVC8TulT0IhZi+RoGvHKXvNhysw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mongodb": { - "version": "0.61.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.61.0.tgz", - "integrity": "sha512-OV3i2DSoY5M/pmLk+68xr5RvkHU8DRB3DKMzYJdwDdcxeLs62tLbkmRyqJZsYf3Ht7j11rq35pHOWLuLzXL7pQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mongoose": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.55.0.tgz", - "integrity": "sha512-5afj0HfF6aM6Nlqgu6/PPHFk8QBfIe3+zF9FGpX76jWPS0/dujoEYn82/XcLSaW5LPUDW8sni+YeK0vTBNri+w==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mysql": { - "version": "0.54.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql/-/instrumentation-mysql-0.54.0.tgz", - "integrity": "sha512-bqC1YhnwAeWmRzy1/Xf9cDqxNG2d/JDkaxnqF5N6iJKN1eVWI+vg7NfDkf52/Nggp3tl1jcC++ptC61BD6738A==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@types/mysql": "2.15.27" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mysql2": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql2/-/instrumentation-mysql2-0.55.0.tgz", - "integrity": "sha512-0cs8whQG55aIi20gnK8B7cco6OK6N+enNhW0p5284MvqJ5EPi+I1YlWsWXgzv/V2HFirEejkvKiI4Iw21OqDWg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.33.0", - "@opentelemetry/sql-common": "^0.41.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-pg": { - "version": "0.61.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.61.0.tgz", - "integrity": "sha512-UeV7KeTnRSM7ECHa3YscoklhUtTQPs6V6qYpG283AB7xpnPGCUCUfECFT9jFg6/iZOQTt3FHkB1wGTJCNZEvPw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.34.0", - "@opentelemetry/sql-common": "^0.41.2", - "@types/pg": "8.15.6", - "@types/pg-pool": "2.0.6" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-redis": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-redis/-/instrumentation-redis-0.57.0.tgz", - "integrity": "sha512-bCxTHQFXzrU3eU1LZnOZQ3s5LURxQPDlU3/upBzlWY77qOI1GZuGofazj3jtzjctMJeBEJhNwIFEgRPBX1kp/Q==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/redis-common": "^0.38.2", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-tedious": { - "version": "0.27.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-tedious/-/instrumentation-tedious-0.27.0.tgz", - "integrity": "sha512-jRtyUJNZppPBjPae4ZjIQ2eqJbcRaRfJkr0lQLHFmOU/no5A6e9s1OHLd5XZyZoBJ/ymngZitanyRRA5cniseA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@types/tedious": "^4.0.14" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-undici": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.19.0.tgz", - "integrity": "sha512-Pst/RhR61A2OoZQZkn6OLpdVpXp6qn3Y92wXa6umfJe9rV640r4bc6SWvw4pPN6DiQqPu2c8gnSSZPDtC6JlpQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.24.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.7.0" - } - }, - "node_modules/@opentelemetry/redis-common": { - "version": "0.38.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/redis-common/-/redis-common-0.38.2.tgz", - "integrity": "sha512-1BCcU93iwSRZvDAgwUxC/DV4T/406SkMfxGqu5ojc3AvNI+I9GhV7v0J1HljsczuuhcnFLYqD5VmwVXfCGHzxA==", - "license": "Apache-2.0", - "engines": { - "node": "^18.19.0 || >=20.6.0" - } - }, - "node_modules/@opentelemetry/resources": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.4.0.tgz", - "integrity": "sha512-RWvGLj2lMDZd7M/5tjkI/2VHMpXebLgPKvBUd9LRasEWR2xAynDwEYZuLvY9P2NGG73HF07jbbgWX2C9oavcQg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.4.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/resources/node_modules/@opentelemetry/core": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.4.0.tgz", - "integrity": "sha512-KtcyFHssTn5ZgDu6SXmUznS80OFs/wN7y6MyFRRcKU6TOw8hNcGxKvt8hsdaLJfhzUszNSjURetq5Qpkad14Gw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/sdk-trace-base": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.4.0.tgz", - "integrity": "sha512-WH0xXkz/OHORDLKqaxcUZS0X+t1s7gGlumr2ebiEgNZQl2b0upK2cdoD0tatf7l8iP74woGJ/Kmxe82jdvcWRw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.4.0", - "@opentelemetry/resources": "2.4.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/sdk-trace-base/node_modules/@opentelemetry/core": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.4.0.tgz", - "integrity": "sha512-KtcyFHssTn5ZgDu6SXmUznS80OFs/wN7y6MyFRRcKU6TOw8hNcGxKvt8hsdaLJfhzUszNSjURetq5Qpkad14Gw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/semantic-conventions": { - "version": "1.38.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.38.0.tgz", - "integrity": "sha512-kocjix+/sSggfJhwXqClZ3i9Y/MI0fp7b+g7kCRm6psy2dsf8uApTRclwG18h8Avm7C9+fnt+O36PspJ/OzoWg==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/@opentelemetry/sql-common": { - "version": "0.41.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/sql-common/-/sql-common-0.41.2.tgz", - "integrity": "sha512-4mhWm3Z8z+i508zQJ7r6Xi7y4mmoJpdvH0fZPFRkWrdp5fq7hhZ2HhYokEOLkfqSMgPR4Z9EyB3DBkbKGOqZiQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.1.0" - } - }, - "node_modules/@prisma/instrumentation": { - "version": "6.19.0", - "resolved": "https://registry.npmjs.org/@prisma/instrumentation/-/instrumentation-6.19.0.tgz", - "integrity": "sha512-QcuYy25pkXM8BJ37wVFBO7Zh34nyRV1GOb2n3lPkkbRYfl4hWl3PTcImP41P0KrzVXfa/45p6eVCos27x3exIg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": ">=0.52.0 <1" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.8" - } - }, - "node_modules/@sentry/core": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/core/-/core-10.34.0.tgz", - "integrity": "sha512-4FFpYBMf0VFdPcsr4grDYDOR87mRu6oCfb51oQjU/Pndmty7UgYo0Bst3LEC/8v0SpytBtzXq+Wx/fkwulBesg==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@sentry/node": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/node/-/node-10.34.0.tgz", - "integrity": "sha512-bEOyH97HuVtWZYAZ5mp0NhYNc+n6QCfiKuLee2P75n2kt4cIPTGvLOSdUwwjllf795uOdKZJuM1IUN0W+YMcVg==", - "license": "MIT", - "dependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^2.2.0", - "@opentelemetry/core": "^2.2.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/instrumentation-amqplib": "0.55.0", - "@opentelemetry/instrumentation-connect": "0.52.0", - "@opentelemetry/instrumentation-dataloader": "0.26.0", - "@opentelemetry/instrumentation-express": "0.57.0", - "@opentelemetry/instrumentation-fs": "0.28.0", - "@opentelemetry/instrumentation-generic-pool": "0.52.0", - "@opentelemetry/instrumentation-graphql": "0.56.0", - "@opentelemetry/instrumentation-hapi": "0.55.0", - "@opentelemetry/instrumentation-http": "0.208.0", - "@opentelemetry/instrumentation-ioredis": "0.56.0", - "@opentelemetry/instrumentation-kafkajs": "0.18.0", - "@opentelemetry/instrumentation-knex": "0.53.0", - "@opentelemetry/instrumentation-koa": "0.57.0", - "@opentelemetry/instrumentation-lru-memoizer": "0.53.0", - "@opentelemetry/instrumentation-mongodb": "0.61.0", - "@opentelemetry/instrumentation-mongoose": "0.55.0", - "@opentelemetry/instrumentation-mysql": "0.54.0", - "@opentelemetry/instrumentation-mysql2": "0.55.0", - "@opentelemetry/instrumentation-pg": "0.61.0", - "@opentelemetry/instrumentation-redis": "0.57.0", - "@opentelemetry/instrumentation-tedious": "0.27.0", - "@opentelemetry/instrumentation-undici": "0.19.0", - "@opentelemetry/resources": "^2.2.0", - "@opentelemetry/sdk-trace-base": "^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0", - "@prisma/instrumentation": "6.19.0", - "@sentry/core": "10.34.0", - "@sentry/node-core": "10.34.0", - "@sentry/opentelemetry": "10.34.0", - "import-in-the-middle": "^2.0.1", - "minimatch": "^9.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@sentry/node-core": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/node-core/-/node-core-10.34.0.tgz", - "integrity": "sha512-FrGfC8GzD1cnZDO3zwQ4cjyoY1ZwNHvZbXSvXRYxpjhXidZhvaPurjgLRSB0xGaFgoemmOp1ufsx/w6fQOGA6Q==", - "license": "MIT", - "dependencies": { - "@apm-js-collab/tracing-hooks": "^0.3.1", - "@sentry/core": "10.34.0", - "@sentry/opentelemetry": "10.34.0", - "import-in-the-middle": "^2.0.1" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/core": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/instrumentation": ">=0.57.1 <1", - "@opentelemetry/resources": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0" - } - }, - "node_modules/@sentry/opentelemetry": { - "version": "10.34.0", - "resolved": "https://registry.npmjs.org/@sentry/opentelemetry/-/opentelemetry-10.34.0.tgz", - "integrity": "sha512-uKuULBOmdVu3bYdD8doMLqKgN0PP3WWtI7Shu11P9PVrhSNT4U9yM9Z6v1aFlQcbrgyg3LynZuXs8lyjt90UbA==", - "license": "MIT", - "dependencies": { - "@sentry/core": "10.34.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/core": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0" - } - }, - "node_modules/@standard-schema/spec": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.0.0.tgz", - "integrity": "sha512-m2bOd0f2RT9k8QJx1JN85cZYyH1RqFBdlwtkSlf4tBDYLCiiZnv1fIIwacK6cqwXavOydf0NPToMQgpKq+dVlA==", - "license": "MIT" - }, - "node_modules/@types/connect": { - "version": "3.4.38", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", - "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/mysql": { - "version": "2.15.27", - "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.27.tgz", - "integrity": "sha512-YfWiV16IY0OeBfBCk8+hXKmdTKrKlwKN1MNKAPBu5JYxLwBEZl7QzeEpGnlZb3VMGJrrGmB84gXiH+ofs/TezA==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/node": { - "version": "24.10.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", - "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", - "license": "MIT", - "dependencies": { - "undici-types": "~7.16.0" - } - }, - "node_modules/@types/pg": { - "version": "8.15.6", - "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.15.6.tgz", - "integrity": "sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "pg-protocol": "*", - "pg-types": "^2.2.0" - } - }, - "node_modules/@types/pg-pool": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@types/pg-pool/-/pg-pool-2.0.6.tgz", - "integrity": "sha512-TaAUE5rq2VQYxab5Ts7WZhKNmuN78Q6PiFonTDdpbx8a1H0M1vhy3rhiMjl+e2iHmogyMw7jZF4FrE6eJUy5HQ==", - "license": "MIT", - "dependencies": { - "@types/pg": "*" - } - }, - "node_modules/@types/tedious": { - "version": "4.0.14", - "resolved": "https://registry.npmjs.org/@types/tedious/-/tedious-4.0.14.tgz", - "integrity": "sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@vercel/oidc": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.0.3.tgz", - "integrity": "sha512-yNEQvPcVrK9sIe637+I0jD6leluPxzwJKx/Haw6F4H77CdDsszUn5V3o96LPziXkSNE2B83+Z3mjqGKBK/R6Gg==", - "license": "Apache-2.0", - "engines": { - "node": ">= 20" - } - }, - "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", - "license": "MIT", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-import-attributes": { - "version": "1.9.5", - "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", - "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", - "license": "MIT", - "peerDependencies": { - "acorn": "^8" - } - }, - "node_modules/ai": { - "version": "5.0.87", - "resolved": "https://registry.npmjs.org/ai/-/ai-5.0.87.tgz", - "integrity": "sha512-9Cjx7o8IY9zAczigX0Tk/BaQwjPe/M6DpEjejKSBNrf8mOPIvyM+pJLqJSC10IsKci3FPsnaizJeJhoetU1Wfw==", - "license": "Apache-2.0", - "dependencies": { - "@ai-sdk/gateway": "2.0.6", - "@ai-sdk/provider": "2.0.0", - "@ai-sdk/provider-utils": "3.0.16", - "@opentelemetry/api": "1.9.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.25.76 || ^4.1.8" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "license": "MIT" - }, - "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/cjs-module-lexer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.2.0.tgz", - "integrity": "sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==", - "license": "MIT" - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/dotenv": { - "version": "17.2.3", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", - "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/eventsource-parser": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz", - "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==", - "license": "MIT", - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/forwarded-parse": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/forwarded-parse/-/forwarded-parse-2.1.2.tgz", - "integrity": "sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==", - "license": "MIT" - }, - "node_modules/import-in-the-middle": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-2.0.4.tgz", - "integrity": "sha512-Al0kMpa0BqfvDnxjxGlab9vdQ0vTDs82TBKrD59X9jReUoPAzSGBb6vGDzMUMFBGyyDF03RpLT4oxGn6BpASzQ==", - "license": "Apache-2.0", - "dependencies": { - "acorn": "^8.15.0", - "acorn-import-attributes": "^1.9.5", - "cjs-module-lexer": "^2.2.0", - "module-details-from-path": "^1.0.4" - } - }, - "node_modules/json-schema": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", - "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", - "license": "(AFL-2.1 OR BSD-3-Clause)" - }, - "node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/module-details-from-path": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", - "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", - "license": "MIT" - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/pg-int8": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", - "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", - "license": "ISC", - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/pg-protocol": { - "version": "1.10.3", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.10.3.tgz", - "integrity": "sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==", - "license": "MIT" - }, - "node_modules/pg-types": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", - "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", - "license": "MIT", - "dependencies": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-array": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", - "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-bytea": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", - "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-date": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", - "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-interval": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", - "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", - "license": "MIT", - "dependencies": { - "xtend": "^4.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-in-the-middle": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-8.0.1.tgz", - "integrity": "sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==", - "license": "MIT", - "dependencies": { - "debug": "^4.3.5", - "module-details-from-path": "^1.0.3" - }, - "engines": { - "node": ">=9.3.0 || >=8.10.0 <9.0.0" - } - }, - "node_modules/undici-types": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", - "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", - "license": "MIT" - }, - "node_modules/xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "license": "MIT", - "engines": { - "node": ">=0.4" - } - }, - "node_modules/zod": { - "version": "4.1.12", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.12.tgz", - "integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==", - "license": "MIT", - "peer": true, - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - } - } -} diff --git a/sdks/js/vercel/package.json b/sdks/js/vercel/package.json deleted file mode 100644 index 0c3df4b..0000000 --- a/sdks/js/vercel/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "@sentry-ai-sdks/vercel", - "version": "1.0.0", - "description": "Sentry integration tests for Vercel AI SDK", - "scripts": { - "test": "echo \"Use the CLI: npm run cli run --sdk js/vercel\"" - }, - "dependencies": { - "@ai-sdk/openai": "2.0.62", - "@sentry/node": "10.34.0", - "ai": "5.0.87", - "dotenv": "17.2.3" - } -} diff --git a/sdks/js/vercel/setup.js b/sdks/js/vercel/setup.js deleted file mode 100644 index a195e51..0000000 --- a/sdks/js/vercel/setup.js +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Setup file for Vercel AI SDK tests - * - * Initializes Sentry with Vercel AI-specific integrations. - */ - -const Sentry = require("@sentry/node"); -const { createTransport } = require("@sentry/core"); -const { config } = require("dotenv"); -const { resolve } = require("path"); -const { createMockTransport } = require("../_test-utils/mock-transport.cjs"); - -// Load environment variables -config({ quiet: true, path: resolve(__dirname, ".env") }); - -// Initialize Sentry. -// NOTE: AI integrations are auto-enabled in the Node.js SDK and -// therefore do not need to be configured explicitly. -Sentry.init({ - dsn: process.env.SENTRY_DSN || "https://public@127.0.0.1/1", - tracesSampleRate: 1.0, - transport: createMockTransport(createTransport), - sendDefaultPii: true, -}); - -module.exports = { Sentry }; diff --git a/sdks/py/_test-utils/README.md b/sdks/py/_test-utils/README.md deleted file mode 100644 index 323f5ea..0000000 --- a/sdks/py/_test-utils/README.md +++ /dev/null @@ -1,264 +0,0 @@ -# Python Test Utilities - -This directory contains Python testing utilities for SDK implementations. - -## Directory Structure - -``` -sdks/py/_test-utils/ -├── test_runner.py # Orchestrates test execution (main helper) -├── fixture_loader.py # Loads JSON fixtures with config overrides -├── validator.py # Validates captured data against fixtures -├── validator.test.py # Tests for validator logic -├── mock_transport.py # Captures Sentry data in-memory -└── README.md # This file -``` - -**Note:** All files use `.py` extension with snake_case naming. No requirements.txt - dependencies come from each SDK's requirements.txt. - -## 🚨 CRITICAL: JavaScript/Python Parity Rule - -**The utilities in `sdks/py/_test-utils/` MUST be kept synchronized with `sdks/js/_test-utils/`.** - -### Why This Matters - -- Same fixtures (JSON) used by both languages -- Same validation logic = consistent behavior -- Same error messages = easier debugging -- Changes to one MUST be mirrored in the other - -### When You Change Test Utils - -**ALWAYS update both JS and Python versions together:** - -1. **If you modify `validator.py`:** - - Update `sdks/js/_test-utils/validator.cjs` with equivalent logic - - **Update `validator.test.py` and `validator.test.cjs` with test cases for the new feature** - - Run both test files: `python3 validator.test.py && node validator.test.cjs` - - Run same fixture through both validators - - Confirm identical error output - -2. **If you modify `test_runner.py`:** - - Update `sdks/js/_test-utils/test-runner.cjs` with equivalent logic - - Test both implementations - - Verify error messages match - -3. **If you add a new helper function:** - - Implement in both languages - - Keep function signatures equivalent (account for camelCase vs snake_case) - - Document any language-specific differences - -## Core Components - -### test_runner.py - -The main helper that orchestrates test execution. Provides: - -- **`run_test_case(testCaseId, testLogic)`** - Main test orchestration function - - Loads SDK config from environment (`SDK_CONFIG`) - - Loads fixture with `$ref` resolution and config overrides applied - - Displays test name from `fixture["name"]` (no hardcoded descriptions) - - Wraps test logic in Sentry span - - Validates captured data against fixture - - Shows SDK path in output: `[py/openai]` - - Returns dict with `main()` and `assert_sentry()` functions - -### fixture_loader.py - -Loads JSON fixtures from `shared/specs/` with SDK-specific overrides: - -- **`load_fixture(test_case_id, framework_type, config_overrides)`** - - Reads fixture from `shared/specs/{test_case_id}/fixture-{framework_type}.json` - - Resolves `$ref` references to shared span definitions (`common-spans.json`) - - Applies config overrides (model names, span attributes) - - Returns fixture with all overrides applied - -**Features:** -- `$ref` syntax: `{ "$ref": "common-spans#/llm_call", "parent": "agent" }` -- Cached common spans for performance -- Override properties merge with referenced spans - -### validator.py - -Validates captured Sentry data against fixture expectations: - -- **`validate_fixture(test_case_id, spans, transactions, events, framework_type, config_overrides)`** - - Main validation function that orchestrates all validation steps - - Checks transactions, spans, attributes, hierarchy, events - - Returns validation result with detailed error messages - -**Internal validation functions (not exported):** -- `validate_transactions()`, `validate_span_counts()`, `validate_events()` - Count validations -- `validate_span_items()` - Matches and validates individual spans -- `validate_span_relationships()` - Validates parent-child hierarchy -- `validate_span_attributes()` - Validates attributes with schema support -- `normalize_op_to_list()`, `format_op_description()` - Helper utilities - -**Supported validation features:** -- Wildcard patterns: `"gpt-4*"`, `"*-mini"`, `"*anthropic*"` -- Pattern-based op matching: `{ "pattern": "gen_ai.*", "not": [...] }` -- Schema validation: `{ "type": "json_array", "min_length": 2, "items_have": [...] }`, `{ "type": "plain_string" }` -- Presence checks: `True` (attribute must exist) -- Order-based span matching: Multiple spans with same op matched in fixture order -- `None` treated as missing (not mismatch) - -**Exports:** Only `validate_fixture` and `attribute_matches` (via `__all__`) - -**Test file:** `validator.test.py` - Run with `python3 validator.test.py` to verify validator logic - -### mock_transport.py - -In-memory Sentry transport for testing: - -- **`create_mock_transport(options)`** - Factory for mock transport -- **`get_mock_transport()`** - Get current transport instance -- **`clear_mock_transport()`** - Clear captured data - -## Usage - -### In SDK Setup Files (setup.py) - -```python -import os -import sys -import sentry_sdk -from pathlib import Path -from dotenv import load_dotenv - -# Add test utils to path (CRITICAL - DO NOT FORGET) -test_utils_path = Path(__file__).parent.parent / "_test-utils" -sys.path.insert(0, str(test_utils_path)) - -from mock_transport import create_mock_transport, get_mock_transport, clear_mock_transport - - -def before_all(): - """Initialize Sentry with mock transport""" - print("🔧 Setting up {Your SDK} tests...") - - # Load environment variables - env_path = Path(__file__).parent.parent.parent.parent / ".env" - load_dotenv(dotenv_path=env_path) - - # Pre-initialize mock transport - from mock_transport import MockTransportCapture - import mock_transport as mt - mt._mock_transport_capture = MockTransportCapture() - - mock_transport_instance = create_mock_transport( - options={"dsn": os.getenv("SENTRY_DSN", "https://public@127.0.0.1/1")} - ) - - # Initialize Sentry - # Note: AI integrations are auto-enabled in Python - no need to manually add them - sentry_sdk.init( - traces_sample_rate=1.0, - transport=mock_transport_instance, - ) - - print(" ✓ Sentry initialized with mock transport") - - -def before_each(): - """Reset test state""" - print(" ↻ Resetting test state...") - clear_mock_transport() - - -def after_each(): - """Clean up after test""" - print(" ✓ Cleaning up...") - - -def after_all(): - """Teardown Sentry""" - print("🧹 Tearing down {Your SDK} tests...") - sentry_sdk.flush(timeout=2.0) - - -def get_mock_sentry_transport(): - """Helper to get mock transport for assertions""" - return get_mock_transport() -``` - -### In SDK Config Files (config.json) - -```json -{ - "sdk_name": "your-sdk", - "framework_type": "low-level", - "overrides": { - "1-simple": { - "model": "your-model-name", - "gen_ai.request.model": "your-model-name", - "gen_ai.response.model": "your-model-name" - } - } -} -``` - -### In Test Case Files (cases/1-simple.py) - -```python -import os -from your_sdk import YourClient -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - prompt = inputs["prompt"] - - # Your SDK-specific test logic here - client = YourClient(api_key=os.getenv("YOUR_API_KEY")) - response = client.generate(model=model, system=system, prompt=prompt) - - if not response.text: - raise Exception("No output returned") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {response.text}") - - return response.text - - -# Framework type is loaded from config.json automatically -test_case = run_test_case("1-simple", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] -``` - -## Important Notes - -### sys.path Setup - -Every Python SDK's `setup.py` MUST include this code to make test utils importable: - -```python -import sys -from pathlib import Path - -test_utils_path = Path(__file__).parent.parent / "_test-utils" -sys.path.insert(0, str(test_utils_path)) -``` - -Test case files will automatically inherit this path setup. - -## Parity Status - -| Component | Python | JavaScript | Status | Notes | -|-----------|--------|------------|--------|-------| -| Test Runner | `test_runner.py` | `test-runner.cjs` | ✅ Synced | Both orchestrate tests correctly | -| Mock Transport | `mock_transport.py` | `mock-transport.cjs` | ✅ Synced | Both capture envelopes correctly | -| Fixture Loader | `fixture_loader.py` | `fixture-loader.cjs` | ✅ Synced | Both support config overrides | -| Fixture Validator | `validator.py` | `validator.cjs` | ✅ Synced | Schema validation, pattern ops, wildcards | -| Validator Tests | `validator.test.py` | `validator.test.cjs` | ✅ Synced | Both test schema validation | - -## See Also - -- [JavaScript Test Utilities](../../js/_test-utils/README.md) - JavaScript equivalent of these utilities -- [Adding SDKs Guide](../README.md) - Step-by-step SDK implementation guide -- [Test Specifications](../../../shared/specs/README.md) - Fixture format & framework types -- [Main Documentation](../../../CLAUDE.md) - Project overview & critical rules diff --git a/sdks/py/_test-utils/fixture_loader.py b/sdks/py/_test-utils/fixture_loader.py deleted file mode 100644 index b48d32d..0000000 --- a/sdks/py/_test-utils/fixture_loader.py +++ /dev/null @@ -1,143 +0,0 @@ -""" -Fixture loader - loads JSON test fixtures from shared/specs/ -""" - -import json -import os -from pathlib import Path -from typing import Dict, Any, Optional -import copy - -# Cache for common spans -_common_spans_cache = None - - -def load_common_spans() -> Dict[str, Any]: - """ - Load common span definitions - - Returns: - Common span definitions - """ - global _common_spans_cache - - if _common_spans_cache is not None: - return _common_spans_cache - - current_dir = Path(__file__).parent - common_spans_path = current_dir / "../../../shared/specs/common-spans.json" - common_spans_path = common_spans_path.resolve() - - if not common_spans_path.exists(): - return {} - - with open(common_spans_path, "r", encoding="utf-8") as f: - _common_spans_cache = json.load(f) - - return _common_spans_cache - - -def resolve_ref(span_item: Dict[str, Any], common_spans: Dict[str, Any]) -> Dict[str, Any]: - """ - Resolve $ref in a span item - - Args: - span_item: Span item that may contain $ref - common_spans: Common span definitions - - Returns: - Resolved span item - """ - if "$ref" not in span_item: - return span_item - - # Parse $ref format: "common-spans#/span_name" - ref = span_item["$ref"] - if not ref.startswith("common-spans#/"): - raise ValueError(f'Invalid $ref format: {ref}. Expected format: "common-spans#/span_name"') - - span_name = ref[len("common-spans#/"):] - common_span = common_spans.get(span_name) - - if not common_span: - raise ValueError(f"Common span not found: {span_name}") - - # Merge common span with overrides from the reference - # Properties in span_item (except $ref) override common span properties - overrides = {k: v for k, v in span_item.items() if k != "$ref"} - return {**common_span, **overrides} - - -def apply_overrides(fixture: Dict[str, Any], overrides: Optional[Dict[str, Any]]) -> Dict[str, Any]: - """ - Apply overrides to a fixture object - - Args: - fixture: The fixture object to modify - overrides: Key-value pairs to override in the fixture - - Returns: - The modified fixture object - """ - if not overrides or len(overrides) == 0: - return fixture - - # Deep clone to avoid mutating original - result = copy.deepcopy(fixture) - - for key, value in overrides.items(): - # Handle special "model" shorthand - applies to inputs.model - if key == "model": - if "inputs" in result: - result["inputs"]["model"] = value - continue - - # Handle dot-notation paths in expectations (e.g., "gen_ai.request.model") - # These override values in required_attributes - if "expectations" in result and "spans" in result["expectations"]: - if "items" in result["expectations"]["spans"]: - for span_item in result["expectations"]["spans"]["items"]: - if "required_attributes" in span_item and key in span_item["required_attributes"]: - span_item["required_attributes"][key] = value - - return result - - -def load_fixture(spec_id: str, variant: str = "agentic", overrides: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: - """ - Load a fixture by spec ID and variant - - Args: - spec_id: The spec ID (e.g., "1-simple", "2-simple-with-error") - variant: The fixture variant (e.g., "agentic", "low-level") - overrides: Optional key-value overrides to apply to the fixture - - Returns: - The parsed fixture object - - Raises: - FileNotFoundError: If fixture file not found - """ - # Fixtures are in shared/specs/{spec_id}/fixture-{variant}.json - # Path from sdks/py/_test-utils/ to shared/specs/ - current_dir = Path(__file__).parent - fixture_path = current_dir / "../../../shared/specs" / spec_id / f"fixture-{variant}.json" - fixture_path = fixture_path.resolve() - - if not fixture_path.exists(): - raise FileNotFoundError(f"Fixture not found: {spec_id} (variant: {variant}) at {fixture_path}") - - with open(fixture_path, "r", encoding="utf-8") as f: - fixture = json.load(f) - - # Resolve $ref references in span items - if "expectations" in fixture and "spans" in fixture["expectations"]: - if "items" in fixture["expectations"]["spans"]: - common_spans = load_common_spans() - fixture["expectations"]["spans"]["items"] = [ - resolve_ref(item, common_spans) - for item in fixture["expectations"]["spans"]["items"] - ] - - # Apply overrides if provided - return apply_overrides(fixture, overrides) diff --git a/sdks/py/_test-utils/mock_transport.py b/sdks/py/_test-utils/mock_transport.py deleted file mode 100644 index 944615c..0000000 --- a/sdks/py/_test-utils/mock_transport.py +++ /dev/null @@ -1,199 +0,0 @@ -""" -Mock Sentry transport for testing - -Captures all Sentry events in memory instead of sending them to Sentry servers. -Provides helpers to query and verify captured events. -""" - -import json -from typing import List, Dict, Any, Optional - - -class MockTransportCapture: - """Captures Sentry envelopes in memory""" - - def __init__(self): - self.envelopes: List[Any] = [] - - def capture(self, envelope: Any) -> None: - """Capture an envelope""" - self.envelopes.append(envelope) - - def clear(self) -> None: - """Clear all captured envelopes""" - self.envelopes = [] - - def get_envelopes(self) -> List[Any]: - """Get all captured envelopes""" - return self.envelopes - - def _parse_envelope_body(self, body: str) -> Dict[str, Any]: - """ - Parse envelope body string into header and items - Envelope format is newline-separated: - Line 1: Envelope headers (JSON) - Line 2: Item 1 headers (JSON) - Line 3: Item 1 payload (JSON) - etc. - """ - lines = [line for line in body.split("\n") if line.strip()] - - if not lines: - return {"headers": {}, "items": []} - - # First line is envelope headers - headers = json.loads(lines[0]) - items = [] - - # Remaining lines are pairs of item headers + item payload - i = 1 - while i < len(lines): - if i + 1 < len(lines): - item_headers = json.loads(lines[i]) - item_payload = json.loads(lines[i + 1]) - items.append({"headers": item_headers, "payload": item_payload}) - i += 2 - else: - break - - return {"headers": headers, "items": items} - - def get_transactions(self) -> List[Dict[str, Any]]: - """Get all captured transactions""" - transactions = [] - - for envelope in self.envelopes: - # Handle Envelope objects (sentry_sdk.envelope.Envelope) - if hasattr(envelope, "items"): - for item in envelope.items: - if hasattr(item, "headers") and item.headers.get("type") == "transaction": - # Get the payload - it might be a Payload object - if hasattr(item, "payload"): - payload = item.payload - # If it's a Payload object, get its data - if hasattr(payload, "json"): - transactions.append(payload.json) - elif hasattr(payload, "bytes"): - import json - transactions.append(json.loads(payload.bytes)) - else: - transactions.append(payload) - # Fallback: Handle string body format (for backwards compatibility) - elif hasattr(envelope, "body") or (isinstance(envelope, dict) and "body" in envelope): - body = envelope.body if hasattr(envelope, "body") else envelope["body"] - if isinstance(body, str): - parsed = self._parse_envelope_body(body) - for item in parsed["items"]: - if item["headers"].get("type") == "transaction": - transactions.append(item["payload"]) - - return transactions - - def get_spans(self) -> List[Dict[str, Any]]: - """Get all captured spans (extracted from transactions)""" - spans = [] - - for transaction in self.get_transactions(): - if "spans" in transaction and isinstance(transaction["spans"], list): - spans.extend(transaction["spans"]) - - return spans - - def get_events(self) -> List[Dict[str, Any]]: - """Get all captured events (errors, messages, etc.)""" - events = [] - - for envelope in self.envelopes: - # Handle Envelope objects (sentry_sdk.envelope.Envelope) - if hasattr(envelope, "items"): - for item in envelope.items: - if hasattr(item, "headers") and item.headers.get("type") == "event": - # Get the payload - it might be a Payload object - if hasattr(item, "payload"): - payload = item.payload - # If it's a Payload object, get its data - if hasattr(payload, "json"): - events.append(payload.json) - elif hasattr(payload, "bytes"): - import json - events.append(json.loads(payload.bytes)) - else: - events.append(payload) - # Fallback: Handle string body format (for backwards compatibility) - elif hasattr(envelope, "body") or (isinstance(envelope, dict) and "body" in envelope): - body = envelope.body if hasattr(envelope, "body") else envelope["body"] - if isinstance(body, str): - parsed = self._parse_envelope_body(body) - for item in parsed["items"]: - if item["headers"].get("type") == "event": - events.append(item["payload"]) - - return events - - -# Global instance -_mock_transport_capture: Optional[MockTransportCapture] = None - - -def create_mock_transport(options: Optional[Dict[str, Any]] = None) -> Any: - """ - Create a mock transport factory (to be passed to sentry_sdk.init) - - Args: - options: Transport options - - Returns: - Mock transport instance - """ - global _mock_transport_capture - from sentry_sdk.transport import Transport - - # Only create if not already set (it should be pre-initialized by setup.py) - if not _mock_transport_capture: - _mock_transport_capture = MockTransportCapture() - - class MockTransport(Transport): - def __init__(self, options): - # Ensure options has required keys for Transport - if not options: - options = {} - if "dsn" not in options: - options["dsn"] = None - Transport.__init__(self, options) - - def capture_event(self, event): - """Capture an event""" - if _mock_transport_capture: - _mock_transport_capture.capture(event) - - def capture_envelope(self, envelope): - """Capture an envelope""" - if _mock_transport_capture: - _mock_transport_capture.capture(envelope) - - def _send_event(self, event): - """Legacy method for sending events""" - if _mock_transport_capture: - _mock_transport_capture.capture(event) - - def _send_envelope(self, envelope): - """Legacy method for sending envelopes""" - if _mock_transport_capture: - _mock_transport_capture.capture(envelope) - - return MockTransport(options) - - -def get_mock_transport() -> MockTransportCapture: - """Get the current mock transport capture instance""" - if not _mock_transport_capture: - raise RuntimeError( - "Mock transport not initialized. Did you call sentry_sdk.init with create_mock_transport?" - ) - return _mock_transport_capture - - -def clear_mock_transport() -> None: - """Clear all captured events""" - if _mock_transport_capture: - _mock_transport_capture.clear() diff --git a/sdks/py/_test-utils/test_runner.py b/sdks/py/_test-utils/test_runner.py deleted file mode 100644 index c04a6fa..0000000 --- a/sdks/py/_test-utils/test_runner.py +++ /dev/null @@ -1,105 +0,0 @@ -""" -Test runner helper - orchestrates test execution with minimal boilerplate -""" - -import asyncio -import sentry_sdk -from fixture_loader import load_fixture -from validator import validate_fixture -from mock_transport import get_mock_transport - - -def run_test_case(spec_id, test_logic): - """ - Run a test case with automatic fixture loading, span wrapping, and validation - - Args: - spec_id: Test spec ID (e.g., "1-simple") - test_logic: Async function containing SDK-specific test logic - - Returns: - Dict with main() and assert_sentry() functions for test runner - """ - - async def main(): - """Main test case entry point""" - import os - import json - - sdk_path = os.getenv("SDK_PATH", "unknown") - - # Load SDK config from environment - sdk_config_json = os.getenv("SDK_CONFIG") - sdk_config = json.loads(sdk_config_json) if sdk_config_json else None - - if not sdk_config or "framework_type" not in sdk_config: - raise Exception( - "SDK_CONFIG with framework_type must be provided via environment variable" - ) - - framework_type = sdk_config["framework_type"] - - # Load config overrides from environment - overrides_json = os.getenv("SDK_CONFIG_OVERRIDES") - overrides = json.loads(overrides_json) if overrides_json else None - - # Load fixture inputs with overrides applied - fixture = load_fixture(spec_id, framework_type, overrides) - - # Log with test name from fixture - print(f"\n [{sdk_path}]") - print(f" Running {spec_id}: {fixture.get('name', spec_id)}") - - # Run test logic - await test_logic(fixture["inputs"]) - - async def assert_sentry(): - """Verify Sentry captured the expected data""" - # Flush and wait for transport - sentry_sdk.flush(timeout=2.0) - await asyncio.sleep(0.05) - - # Load SDK config from environment - import os - import json - sdk_config_json = os.getenv("SDK_CONFIG") - sdk_config = json.loads(sdk_config_json) if sdk_config_json else None - - if not sdk_config or "framework_type" not in sdk_config: - raise Exception( - "SDK_CONFIG with framework_type must be provided via environment variable" - ) - - framework_type = sdk_config["framework_type"] - - # Load config overrides from environment - overrides_json = os.getenv("SDK_CONFIG_OVERRIDES") - overrides = json.loads(overrides_json) if overrides_json else None - - # Get transport data - transport = get_mock_transport() - spans = transport.get_spans() - transactions = transport.get_transactions() - events = transport.get_events() - - print( - f" Captured: {len(spans)} spans, {len(transactions)} transactions, {len(events)} events" - ) - - # Validate with overrides - result = validate_fixture( - spec_id, spans, transactions, events, framework_type, overrides - ) - - if not result["passed"]: - print(" ✗ Validation failed:") - for error in result["errors"]: - print(f" - {error}") - raise Exception( - f"Fixture validation failed:\n" + "\n".join(result["errors"]) - ) - - print(" ✓ All fixture validations passed") - print(f" ✓ {spec_id} completed") - - return {"main": main, "assert_sentry": assert_sentry} diff --git a/sdks/py/_test-utils/validator.py b/sdks/py/_test-utils/validator.py deleted file mode 100644 index 0a6cd6f..0000000 --- a/sdks/py/_test-utils/validator.py +++ /dev/null @@ -1,790 +0,0 @@ -""" -Fixture validator - validates captured Sentry data against fixtures - -Includes assertion helpers for querying and verifying spans -""" - -import json -from typing import List, Dict, Any -from fixture_loader import load_fixture - -# Public API -__all__ = ["validate_fixture", "attribute_matches"] - - -# ============================================================================ -# ASSERTION HELPERS -# ============================================================================ - - -def format_op_description(op: Any) -> str: - """ - Format an op specification as a human-readable description - - Args: - op: Operation specification (string, list, or dict) - - Returns: - Human-readable description - """ - if isinstance(op, dict): - not_list = op.get("not", []) - return f"{op.get('pattern')} (excluding: {', '.join(not_list)})" - elif isinstance(op, list): - return " or ".join(op) - else: - return op - - -def normalize_op_to_list(op: Any, spans: List[Dict[str, Any]]) -> List[str]: - """ - Normalize an op specification to a list of operation names - - Args: - op: Operation specification (string, list, or dict with pattern/not) - spans: Available spans (needed for pattern matching) - - Returns: - List of operation names - """ - if isinstance(op, dict): - # Object format: { "pattern": "gen_ai.*", "not": ["gen_ai.invoke_agent", ...] } - pattern = op.get("pattern") - not_list = op.get("not", []) - - # Get all unique op values from spans that match the pattern but not in the exclusion list - matching_ops = set() - for s in spans: - span_op = s.get("op") - if ( - span_op - and matches_pattern(span_op, pattern) - and span_op not in not_list - ): - matching_ops.add(span_op) - - return list(matching_ops) - elif isinstance(op, str): - return [op] - else: - return op - - -def validate_span_attributes( - span: Dict[str, Any], required_attributes: Dict[str, Any] -) -> Dict[str, List]: - """ - Validate span attributes and collect errors - - Args: - span: The span to validate - required_attributes: Required attributes to check - - Returns: - Dict with 'missing' and 'mismatched' arrays - """ - errors = {"missing": [], "mismatched": []} - - for attr, expected_value in required_attributes.items(): - if expected_value is True: - # Just check presence - if not has_attribute(span, attr): - errors["missing"].append(attr) - else: - # Check value matches - if not attribute_matches(span, attr, expected_value): - actual_value = get_attribute(span, attr) - # Treat None as missing, not mismatch - if actual_value is None: - errors["missing"].append(attr) - else: - errors["mismatched"].append( - { - "attr": attr, - "expected": expected_value, - "actual": actual_value, - } - ) - - return errors - - -def get_attribute(span: Dict[str, Any], attribute_name: str) -> Any: - """ - Get an attribute value from a span - First checks span.data for the attribute, then checks span directly - - Args: - span: The span to check - attribute_name: Name of the attribute (e.g., "gen_ai.request.model") - - Returns: - The attribute value, or None if not found - """ - if not span: - return None - - # First check in span.data (where Sentry stores span attributes) - if "data" in span and isinstance(span["data"], dict): - if attribute_name in span["data"]: - return span["data"][attribute_name] - - # Then check directly on span using dot notation - parts = attribute_name.split(".") - current = span - - for part in parts: - if current and isinstance(current, dict) and part in current: - current = current[part] - else: - return None - - return current - - -def has_attribute(span: Dict[str, Any], attribute_name: str) -> bool: - """Check if a span has an attribute (regardless of value)""" - return get_attribute(span, attribute_name) is not None - - -def matches_pattern(actual_value: Any, pattern: Any) -> bool: - """ - Check if a value matches a pattern with wildcard support - - Wildcard patterns: - - "foo*" matches any string that begins with "foo" - - "*foo" matches any string that ends with "foo" - - "*foo*" matches any string that contains "foo" - - "foo" matches exactly "foo" (no wildcards) - - Args: - actual_value: The actual value to test - pattern: The expected value or pattern (may contain wildcards) - - Returns: - True if the value matches the pattern - """ - # If pattern is not a string, use strict equality - if not isinstance(pattern, str): - return actual_value == pattern - - # Convert actual value to string for pattern matching - actual_str = str(actual_value) - - # Check for wildcard patterns - if "*" in pattern: - # *foo* - contains - if pattern.startswith("*") and pattern.endswith("*"): - substring = pattern[1:-1] - # If substring is empty (pattern is "*" or "**"), no match - if substring == "" or substring == "*": - return False - return substring in actual_str - # foo* - starts with - elif pattern.endswith("*"): - prefix = pattern[:-1] - # If prefix is empty (pattern is just "*"), no match - if prefix == "": - return False - return actual_str.startswith(prefix) - # *foo - ends with - elif pattern.startswith("*"): - suffix = pattern[1:] - # If suffix is empty (pattern is just "*"), no match - if suffix == "": - return False - return actual_str.endswith(suffix) - - # No wildcards, use strict equality - return actual_value == pattern - - -def validate_schema( - attr_value: Any, schema: Dict[str, Any], span: Dict[str, Any] = None -) -> bool: - """ - Validate an attribute against a schema object - - Args: - attr_value: The actual attribute value - schema: Schema object with validation rules - span: The span object (needed for cross-attribute constraints like lte) - - Returns: - True if value matches schema - - Supported schema formats: - - {"type": "json_array", "min_length": 2, "items_have": ["role", "content"]} - - {"type": "json_array", "length": 2, "items_have": ["role"]} - - {"type": "plain_string", "min_length": 1, "pattern": "*hello*"} - - {"type": "number", "lte": "other.attribute.name"} - value must be <= other attribute - """ - if not schema or not isinstance(schema, dict): - return False - - # Handle plain_string type - if schema.get("type") == "plain_string": - # Must be a string - if not isinstance(attr_value, str): - return False - - # Must NOT be valid JSON - try: - json.loads(attr_value) - return False # It's valid JSON, so it's not a plain string - except (json.JSONDecodeError, ValueError): - # Good - not JSON, it's a plain string - pass - - # Validate min_length - if "min_length" in schema and len(attr_value) < schema["min_length"]: - return False - - # Validate max_length - if "max_length" in schema and len(attr_value) > schema["max_length"]: - return False - - # Validate pattern - if "pattern" in schema and not matches_pattern(attr_value, schema["pattern"]): - return False - - return True - - # Handle json_array type - if schema.get("type") == "json_array": - # Store the original string for 'contains' check - original_string = ( - attr_value if isinstance(attr_value, str) else json.dumps(attr_value) - ) - - # Parse JSON if it's a string - if isinstance(attr_value, str): - try: - parsed = json.loads(attr_value) - except (json.JSONDecodeError, ValueError): - return False # Not valid JSON - else: - parsed = attr_value - - # Check if it's an array - if not isinstance(parsed, list): - return False - - # Validate length - if "length" in schema and len(parsed) != schema["length"]: - return False - - if "min_length" in schema and len(parsed) < schema["min_length"]: - return False - - if "max_length" in schema and len(parsed) > schema["max_length"]: - return False - - # Validate length_lte (array length must be <= another attribute's value) - if "length_lte" in schema and span is not None: - other_value = get_attribute(span, schema["length_lte"]) - # Only validate if the other attribute exists and is a number - if other_value is not None and isinstance(other_value, (int, float)): - if len(parsed) > other_value: - return False - - # Validate contains (raw JSON string must contain this substring) - if "contains" in schema: - if schema["contains"] not in original_string: - return False - - # Validate items_have (each item must have these properties) - if "items_have" in schema and isinstance(schema["items_have"], list): - for item in parsed: - if not isinstance(item, dict): - return False - for required_prop in schema["items_have"]: - if required_prop not in item: - return False - - return True - - # Handle number type with constraints - if schema.get("type") == "number": - # Must be a number - if not isinstance(attr_value, (int, float)): - return False - - # Validate lte (less than or equal to another attribute) - if "lte" in schema and span is not None: - other_value = get_attribute(span, schema["lte"]) - # Only validate if the other attribute exists and is a number - if other_value is not None and isinstance(other_value, (int, float)): - if attr_value > other_value: - return False - - return True - - # Unknown schema type - return False - - -def attribute_matches(span: Dict[str, Any], attribute_name: str, value: Any) -> bool: - """Check if a span has an attribute with a specific value""" - attr_value = get_attribute(span, attribute_name) - - # Check if value is a schema object with optional flag - if isinstance(value, dict) and "type" in value: - # If attribute is missing and schema marks it as optional, that's OK - if attr_value is None and value.get("optional") is True: - return True - if attr_value is None: - return False - return validate_schema(attr_value, value, span) - - # For non-schema values, missing attribute means no match - if attr_value is None: - return False - - # Otherwise use pattern matching - return matches_pattern(attr_value, value) - - -def contains_attributes(span: Dict[str, Any], attributes: Dict[str, Any]) -> bool: - """ - Check if a span contains multiple attributes - - Args: - span: The span to check - attributes: Dict mapping attribute names to expected values - - If value is True, only checks attribute presence - - Otherwise checks attribute matches the value exactly - - Returns: - True if all attributes match - """ - for attr_name, expected_value in attributes.items(): - if expected_value is True: - if not has_attribute(span, attr_name): - return False - else: - if not attribute_matches(span, attr_name, expected_value): - return False - - return True - - -def get_span( - spans: List[Dict[str, Any]], - op: Any, - required_attributes: Dict[str, Any] = None, - used_spans: set = None, -) -> Dict[str, Any]: - """ - Get a single span by operation name(s) and/or attributes - Raises if zero or more than one span is found - - Args: - spans: List of span objects - op: Operation name (string), list of operation names, or dict with pattern/not - required_attributes: Optional dict of attributes to filter by - used_spans: Set of span IDs already used (for matching multiple spans in order) - - Returns: - The matching span - - Raises: - ValueError: If zero or multiple spans found - """ - # Normalize op to a list of operation names - op_list = normalize_op_to_list(op, spans) - - # Filter by operation name(s) - matching = [s for s in spans if s.get("op") in op_list] - - # Exclude already-used spans if used_spans is provided - if used_spans is not None: - matching = [s for s in matching if s.get("span_id") not in used_spans] - - # Further filter by attributes if specified - if required_attributes: - matching = [s for s in matching if contains_attributes(s, required_attributes)] - - if len(matching) == 0: - op_desc = format_op_description(op) - - # Check if any spans with the op exist (without attribute filtering) - spans_with_op = [s for s in spans if s.get("op") in op_list] - - if len(spans_with_op) == 0: - # No spans with that op at all - error_msg = f'No span found with op="{op_desc}"' - error_msg += "\n Available spans:" - for i, s in enumerate(spans, 1): - error_msg += f'\n {i}. op="{s.get("op")}"' - raise ValueError(error_msg) - else: - # Spans with that op exist, but don't match required attributes - import os - - is_verbose = os.getenv("SENTRY_AI_TEST_VERBOSE") == "true" - error_msg = ( - f'Found span with op="{op_desc}" but missing required attributes' - ) - - if required_attributes: - span = spans_with_op[0] - span_data = span.get("data", {}) - - # Concise mode: Just show what's missing/mismatched - if not is_verbose: - missing = [] - mismatched = [] - - for attr, expected_val in required_attributes.items(): - actual_val = get_attribute(span, attr) - - if actual_val is None: - missing.append(attr) - elif expected_val is not True and actual_val != expected_val: - mismatched.append( - f"{attr} (expected: {repr(expected_val)}, got: {repr(actual_val)})" - ) - - if missing: - error_msg += f"\n Missing: {', '.join(missing)}" - if mismatched: - error_msg += f"\n Mismatched: {', '.join(mismatched)}" - error_msg += "\n (run with --verbose for full details)" - else: - # Verbose mode: Show everything - error_msg += "\n Required attributes:" - for attr, val in required_attributes.items(): - val_str = "(any value)" if val is True else repr(val) - error_msg += f"\n - {attr}: {val_str}" - - error_msg += "\n Span's actual attributes:" - if span_data: - for key, value in span_data.items(): - error_msg += f"\n - {key}: {repr(value)}" - else: - error_msg += "\n (no attributes)" - - raise ValueError(error_msg) - - if len(matching) > 1: - # If used_spans is provided, we're matching in order - return first match - if used_spans is not None: - return matching[0] - - # Otherwise, multiple matches is an error - op_desc = format_op_description(op) - error_msg = ( - f'Found {len(matching)} spans matching op="{op_desc}", expected exactly 1' - ) - - error_msg += "\n Matching spans:" - for i, s in enumerate(matching, 1): - span_id = s.get("span_id", "?")[:8] - error_msg += f'\n {i}. op="{s.get("op")}" span_id={span_id}' - - raise ValueError(error_msg) - - return matching[0] - - -def get_spans(spans: List[Dict[str, Any]], op: str) -> List[Dict[str, Any]]: - """ - Get all spans matching an operation name - - Args: - spans: List of span objects - op: Operation name to search for - - Returns: - List of matching spans (may be empty) - """ - return [s for s in spans if s.get("op") == op] - - -def is_child_of(child_span: Dict[str, Any], parent_span: Dict[str, Any]) -> bool: - """ - Check if one span is a child of another - - Args: - child_span: The potential child span - parent_span: The potential parent span - - Returns: - True if child_span is a child of parent_span - """ - if not child_span or not parent_span: - return False - - return child_span.get("parent_span_id") == parent_span.get("span_id") - - -# ============================================================================ -# VALIDATOR -# ============================================================================ - - -def validate_transactions( - transactions: List[Dict[str, Any]], expectations: Dict[str, Any], errors: List[str] -) -> None: - """ - Validate transaction count - - Args: - transactions: Captured transactions - expectations: Fixture expectations - errors: Error list to append to - """ - if "transactions" in expectations: - min_count = expectations["transactions"].get("min_count") - if min_count is not None and len(transactions) < min_count: - errors.append( - f"Expected at least {min_count} transaction(s), got {len(transactions)}" - ) - - -def validate_span_counts( - spans: List[Dict[str, Any]], expectations: Dict[str, Any], errors: List[str] -) -> None: - """ - Validate span count - - Args: - spans: Captured spans - expectations: Fixture expectations - errors: Error list to append to - """ - if "spans" in expectations: - count = expectations["spans"].get("count") - min_count = expectations["spans"].get("min_count") - min_span_count = min_count if min_count is not None else count - if min_span_count is not None and len(spans) < min_span_count: - errors.append( - f"Expected at least {min_span_count} span(s), got {len(spans)}" - ) - - -def validate_events( - events: List[Dict[str, Any]], expectations: Dict[str, Any], errors: List[str] -) -> None: - """ - Validate events - - Args: - events: Captured events - expectations: Fixture expectations - errors: Error list to append to - """ - if "events" in expectations: - error_count = expectations["events"].get("error_count") - if error_count is not None: - actual_error_count = len([e for e in events if e.get("level") == "error"]) - if actual_error_count != error_count: - errors.append( - f"Expected {error_count} error event(s), got {actual_error_count}" - ) - - -def validate_span_relationships( - items: List[Dict[str, Any]], span_map: Dict[str, Dict[str, Any]], errors: List[str] -) -> None: - """ - Validate parent-child relationships between spans - - Args: - items: Span item expectations from fixture - span_map: Dict of fixture ID to matched span - errors: Error list to append to - """ - for item_expectation in items: - if "parent" in item_expectation: - child_span = span_map.get(item_expectation["id"]) - parent_span = span_map.get(item_expectation["parent"]) - - if child_span and parent_span: - if not is_child_of(child_span, parent_span): - errors.append( - f'Span with op="{format_op_description(item_expectation["op"])}" should be child of ' - f'span with id="{item_expectation["parent"]}"' - ) - - -def validate_span_items( - spans: List[Dict[str, Any]], items: List[Dict[str, Any]], errors: List[str] -) -> Dict[str, Dict[str, Any]]: - """ - Validate individual span items from fixture expectations - - Args: - spans: Captured spans - items: Span item expectations from fixture - errors: Error list to append to - - Returns: - Dict of fixture ID to matched span - """ - span_map = {} - span_errors = {} - used_spans = set() - - # Match each expected span - for item_expectation in items: - fixture_id = item_expectation["id"] - expected_op = format_op_description(item_expectation["op"]) - - try: - # Get span by operation and attributes (pass used_spans to match in order) - required_attrs = item_expectation.get("required_attributes") - span = get_span(spans, item_expectation["op"], required_attrs, used_spans) - span_map[fixture_id] = span - - # Mark this span as used - if span.get("span_id"): - used_spans.add(span["span_id"]) - - # Validate required attributes and collect errors - if required_attrs: - if fixture_id not in span_errors: - span_errors[fixture_id] = { - "expected_op": expected_op, - "actual_op": span.get("op"), - "missing": [], - "mismatched": [], - } - - span_error = span_errors[fixture_id] - - # Validate attributes and collect errors - attr_errors = validate_span_attributes(span, required_attrs) - span_error["missing"].extend(attr_errors["missing"]) - span_error["mismatched"].extend(attr_errors["mismatched"]) - except Exception as e: - error_msg = str(e) - # get_span threw an error - check if it's about missing attributes or missing span - if "but missing required attributes" in error_msg: - # Span exists but has attribute issues - extract the details - required_attrs = item_expectation.get("required_attributes") - if required_attrs: - # Find the span by op only (without attribute filtering) - op_list = normalize_op_to_list(item_expectation["op"], spans) - matching_span = next( - (s for s in spans if s.get("op") in op_list), None - ) - - if matching_span: - if fixture_id not in span_errors: - span_errors[fixture_id] = { - "expected_op": expected_op, - "actual_op": matching_span.get("op"), - "missing": [], - "mismatched": [], - } - - span_error = span_errors[fixture_id] - - # Validate attributes and collect errors - attr_errors = validate_span_attributes( - matching_span, required_attrs - ) - span_error["missing"].extend(attr_errors["missing"]) - span_error["mismatched"].extend(attr_errors["mismatched"]) - elif "No span found with op=" in error_msg: - # Span doesn't exist at all - if fixture_id not in span_errors: - span_errors[fixture_id] = { - "expected_op": expected_op, - "actual_op": None, - "missing": [], - "mismatched": [], - "not_found": True, - } - else: - # Other error - just append it - errors.append(error_msg) - - # Format span errors in a structured way - for fixture_id, error_details in span_errors.items(): - if error_details.get("not_found"): - errors.append( - f" {fixture_id} (expected: {error_details['expected_op']}): span not found" - ) - elif error_details["missing"] or error_details["mismatched"]: - error_msg = f" {fixture_id} ({error_details['actual_op']}):" - - for attr in error_details["missing"]: - error_msg += f"\n {attr}: missing" - - for mismatch in error_details["mismatched"]: - error_msg += f"\n {mismatch['attr']}: mismatch (expected: {repr(mismatch['expected'])}, got: {repr(mismatch['actual'])})" - - errors.append(error_msg) - - return span_map - - -def validate_fixture( - spec_id: str, - spans: List[Dict[str, Any]], - transactions: List[Dict[str, Any]], - events: List[Dict[str, Any]] = None, - variant: str = "agentic", - overrides: Dict[str, Any] = None, -) -> Dict[str, Any]: - """ - Validate captured Sentry data against a fixture - - Args: - spec_id: The spec ID (e.g., "1-simple") - spans: Captured spans - transactions: Captured transactions - events: Captured events (optional) - variant: The fixture variant (e.g., "agentic", "low-level") - overrides: Optional SDK config overrides to apply to fixture expectations - - Returns: - Validation result dict with 'passed' and 'errors' keys - """ - if events is None: - events = [] - - # Load fixture with overrides applied - fixture = load_fixture(spec_id, variant, overrides) - errors = [] - - # Log captured spans in verbose mode - import os - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print("\n === Captured Spans (Verbose) ===") - if len(spans) == 0: - print(" No spans captured") - else: - for index, span in enumerate(spans): - print(f" Span {index + 1}:") - print(f" op: {span.get('op', 'N/A')}") - print(f" description: {span.get('description', 'N/A')}") - print(f" span_id: {span.get('span_id', 'N/A')}") - print(f" parent_span_id: {span.get('parent_span_id', 'N/A')}") - if span.get("data") and len(span["data"]) > 0: - print(f" data keys: {', '.join(span['data'].keys())}") - print(" === End Captured Spans ===\n") - - # Validate transactions - validate_transactions(transactions, fixture["expectations"], errors) - - # Validate span counts - validate_span_counts(spans, fixture["expectations"], errors) - - # Validate individual spans and relationships - if ( - "spans" in fixture["expectations"] - and "items" in fixture["expectations"]["spans"] - ): - items = fixture["expectations"]["spans"]["items"] - span_map = validate_span_items(spans, items, errors) - validate_span_relationships(items, span_map, errors) - - # Validate events - validate_events(events, fixture["expectations"], errors) - - return {"passed": len(errors) == 0, "errors": errors, "fixture": fixture} diff --git a/sdks/py/_test-utils/validator.test.py b/sdks/py/_test-utils/validator.test.py deleted file mode 100644 index 07e17d4..0000000 --- a/sdks/py/_test-utils/validator.test.py +++ /dev/null @@ -1,618 +0,0 @@ -""" -Simple test for validator - verifies schema validation works -""" - -import sys -import json -from pathlib import Path - -# Add test utils to path -test_utils_path = Path(__file__).parent -sys.path.insert(0, str(test_utils_path)) - -from validator import attribute_matches - -print("\n=== Python Validator Schema Tests ===\n") - -passed = 0 -failed = 0 - -# Test 1: JSON array with correct length -span = { - "data": { - "gen_ai.request.messages": json.dumps( - [{"role": "system", "content": "Hello"}, {"role": "user", "content": "Hi"}] - ) - } -} - -schema = {"type": "json_array", "length": 2, "items_have": ["role", "content"]} -result = attribute_matches(span, "gen_ai.request.messages", schema) - -if result == True: - print("✓ Test 1: Exact length match (2 == 2)") - passed += 1 -else: - print("✗ Test 1: FAILED - Should match length 2") - failed += 1 - -# Test 2: JSON array with insufficient length -span = { - "data": {"gen_ai.request.messages": json.dumps([{"role": "user", "content": "Hi"}])} -} - -schema = {"type": "json_array", "min_length": 2, "items_have": ["role"]} -result = attribute_matches(span, "gen_ai.request.messages", schema) - -if result == False: - print("✓ Test 2: Min length violation (1 < 2)") - passed += 1 -else: - print("✗ Test 2: FAILED - Should reject length < min_length") - failed += 1 - -# Test 3: Missing required property in items -span = { - "data": { - "gen_ai.request.messages": json.dumps( - [ - {"role": "user"}, # Missing 'content' - {"role": "system", "content": "Hi"}, - ] - ) - } -} - -schema = {"type": "json_array", "items_have": ["role", "content"]} -result = attribute_matches(span, "gen_ai.request.messages", schema) - -if result == False: - print("✓ Test 3: Missing required property in item") - passed += 1 -else: - print('✗ Test 3: FAILED - Should detect missing "content" property') - failed += 1 - -# Test 4: Regular string matching (backward compatibility) -span = {"data": {"gen_ai.request.model": "gpt-4"}} -result = attribute_matches(span, "gen_ai.request.model", "gpt-4") - -if result == True: - print("✓ Test 4: Regular string matching still works") - passed += 1 -else: - print("✗ Test 4: FAILED - String matching broken") - failed += 1 - -# Test 5: Wildcard pattern matching (backward compatibility) -span = {"data": {"gen_ai.response.model": "gpt-4-turbo-preview"}} -result = attribute_matches(span, "gen_ai.response.model", "gpt-4*") - -if result == True: - print("✓ Test 5: Wildcard pattern matching still works") - passed += 1 -else: - print("✗ Test 5: FAILED - Wildcard matching broken") - failed += 1 - -# Test 6: Plain string (not JSON) -span = {"data": {"gen_ai.response.text": "Hello world"}} -schema = {"type": "plain_string", "min_length": 1} -result = attribute_matches(span, "gen_ai.response.text", schema) - -if result == True: - print("✓ Test 6: Plain string validation passes") - passed += 1 -else: - print("✗ Test 6: FAILED - Plain string should pass") - failed += 1 - -# Test 7: JSON string rejected as plain_string -span = {"data": {"gen_ai.response.text": "[1, 2, 3]"}} -schema = {"type": "plain_string"} -result = attribute_matches(span, "gen_ai.response.text", schema) - -if result == False: - print("✓ Test 7: JSON string rejected as plain_string") - passed += 1 -else: - print("✗ Test 7: FAILED - Should reject JSON strings") - failed += 1 - -# Test 8: Plain string with pattern -span = {"data": {"gen_ai.system": "You are a helpful assistant"}} -schema = {"type": "plain_string", "pattern": "*helpful*"} -result = attribute_matches(span, "gen_ai.system", schema) - -if result == True: - print("✓ Test 8: Plain string with pattern match") - passed += 1 -else: - print("✗ Test 8: FAILED - Pattern should match") - failed += 1 - -# Test 9: Plain string min_length violation -span = {"data": {"gen_ai.response.text": "Hi"}} -schema = {"type": "plain_string", "min_length": 10} -result = attribute_matches(span, "gen_ai.response.text", schema) - -if result == False: - print("✓ Test 9: Plain string min_length violation (2 < 10)") - passed += 1 -else: - print("✗ Test 9: FAILED - Should reject string shorter than min_length") - failed += 1 - -# ============================================================================ -# Number Schema with lte Constraint Tests -# ============================================================================ - -print("\n--- Number Schema with lte Constraint Tests ---\n") - -# Test 10: Valid lte constraint (50 <= 100) -span = { - "data": {"gen_ai.usage.input_tokens": 100, "gen_ai.usage.input_tokens.cached": 50} -} -schema = {"type": "number", "lte": "gen_ai.usage.input_tokens"} -result = attribute_matches(span, "gen_ai.usage.input_tokens.cached", schema) - -if result == True: - print("✓ Test 10: Valid lte constraint (50 <= 100)") - passed += 1 -else: - print("✗ Test 10: FAILED - Should pass when value <= other") - failed += 1 - -# Test 11: Invalid lte constraint (100 > 50) -span = { - "data": {"gen_ai.usage.input_tokens": 50, "gen_ai.usage.input_tokens.cached": 100} -} -schema = {"type": "number", "lte": "gen_ai.usage.input_tokens"} -result = attribute_matches(span, "gen_ai.usage.input_tokens.cached", schema) - -if result == False: - print("✓ Test 11: Invalid lte constraint detected (100 > 50)") - passed += 1 -else: - print("✗ Test 11: FAILED - Should fail when value > other") - failed += 1 - -# Test 12: Edge case - lte with equal values (100 <= 100) -span = { - "data": {"gen_ai.usage.input_tokens": 100, "gen_ai.usage.input_tokens.cached": 100} -} -schema = {"type": "number", "lte": "gen_ai.usage.input_tokens"} -result = attribute_matches(span, "gen_ai.usage.input_tokens.cached", schema) - -if result == True: - print("✓ Test 12: Edge case - lte with equal values (100 <= 100)") - passed += 1 -else: - print("✗ Test 12: FAILED - Should pass when value == other") - failed += 1 - -# Test 13: lte constraint passes when other attribute is missing -span = {"data": {"gen_ai.usage.input_tokens.cached": 100}} -schema = {"type": "number", "lte": "gen_ai.usage.input_tokens"} -result = attribute_matches(span, "gen_ai.usage.input_tokens.cached", schema) - -if result == True: - print("✓ Test 13: lte passes when other attribute is missing") - passed += 1 -else: - print("✗ Test 13: FAILED - Should pass when other attribute is missing") - failed += 1 - -# Test 14: Number schema fails for non-number values -span = { - "data": { - "gen_ai.usage.input_tokens": 100, - "gen_ai.usage.input_tokens.cached": "not a number", - } -} -schema = {"type": "number", "lte": "gen_ai.usage.input_tokens"} -result = attribute_matches(span, "gen_ai.usage.input_tokens.cached", schema) - -if result == False: - print("✓ Test 14: Number schema fails for non-number values") - passed += 1 -else: - print("✗ Test 14: FAILED - Should fail for non-number values") - failed += 1 - -# Test 15: Number schema without lte constraint (just type check) -span = {"data": {"gen_ai.usage.input_tokens": 100}} -schema = {"type": "number"} -result = attribute_matches(span, "gen_ai.usage.input_tokens", schema) - -if result == True: - print("✓ Test 15: Number schema without lte constraint passes") - passed += 1 -else: - print("✗ Test 15: FAILED - Should pass when only checking type") - failed += 1 - -# Test 16: Valid lte for output tokens reasoning (150 <= 200) -span = { - "data": { - "gen_ai.usage.output_tokens": 200, - "gen_ai.usage.output_tokens.reasoning": 150, - } -} -schema = {"type": "number", "lte": "gen_ai.usage.output_tokens"} -result = attribute_matches(span, "gen_ai.usage.output_tokens.reasoning", schema) - -if result == True: - print("✓ Test 16: Valid lte for output tokens reasoning (150 <= 200)") - passed += 1 -else: - print("✗ Test 16: FAILED - Should pass when value <= other") - failed += 1 - -# Test 17: Invalid lte for output tokens reasoning (150 > 100) -span = { - "data": { - "gen_ai.usage.output_tokens": 100, - "gen_ai.usage.output_tokens.reasoning": 150, - } -} -schema = {"type": "number", "lte": "gen_ai.usage.output_tokens"} -result = attribute_matches(span, "gen_ai.usage.output_tokens.reasoning", schema) - -if result == False: - print("✓ Test 17: Invalid lte for output tokens reasoning detected (150 > 100)") - passed += 1 -else: - print("✗ Test 17: FAILED - Should fail when value > other") - failed += 1 - -# ============================================================================ -# Optional Schema Attribute Tests -# ============================================================================ - -print("\n--- Optional Schema Attribute Tests ---\n") - -# Test 18: Optional attribute missing - should pass -span = { - "data": { - "gen_ai.usage.input_tokens": 100 - # gen_ai.usage.input_tokens.cached is NOT present - } -} -schema = {"type": "number", "lte": "gen_ai.usage.input_tokens", "optional": True} -result = attribute_matches(span, "gen_ai.usage.input_tokens.cached", schema) - -if result == True: - print("✓ Test 18: Optional attribute missing - passes") - passed += 1 -else: - print("✗ Test 18: FAILED - Should pass when optional attribute is missing") - failed += 1 - -# Test 19: Optional attribute present and valid - should pass -span = { - "data": { - "gen_ai.usage.input_tokens": 100, - "gen_ai.usage.input_tokens.cached": 50, - } -} -schema = {"type": "number", "lte": "gen_ai.usage.input_tokens", "optional": True} -result = attribute_matches(span, "gen_ai.usage.input_tokens.cached", schema) - -if result == True: - print("✓ Test 19: Optional attribute present and valid (50 <= 100) - passes") - passed += 1 -else: - print( - "✗ Test 19: FAILED - Should pass when optional attribute is present and valid" - ) - failed += 1 - -# Test 20: Optional attribute present but invalid - should fail -span = { - "data": { - "gen_ai.usage.input_tokens": 50, - "gen_ai.usage.input_tokens.cached": 100, # Invalid: 100 > 50 - } -} -schema = {"type": "number", "lte": "gen_ai.usage.input_tokens", "optional": True} -result = attribute_matches(span, "gen_ai.usage.input_tokens.cached", schema) - -if result == False: - print("✓ Test 20: Optional attribute present but invalid (100 > 50) - fails") - passed += 1 -else: - print( - "✗ Test 20: FAILED - Should fail when optional attribute is present but invalid" - ) - failed += 1 - -# Test 21: Required (non-optional) attribute missing - should fail -span = { - "data": { - "gen_ai.usage.input_tokens": 100 - # gen_ai.usage.input_tokens.cached is NOT present - } -} -schema = {"type": "number", "lte": "gen_ai.usage.input_tokens"} # No optional: True -result = attribute_matches(span, "gen_ai.usage.input_tokens.cached", schema) - -if result == False: - print("✓ Test 21: Required (non-optional) attribute missing - fails") - passed += 1 -else: - print("✗ Test 21: FAILED - Should fail when required attribute is missing") - failed += 1 - -# Test 22: Optional with json_array type - missing attribute -span = { - "data": { - # gen_ai.request.messages is NOT present - } -} -schema = {"type": "json_array", "min_length": 1, "optional": True} -result = attribute_matches(span, "gen_ai.request.messages", schema) - -if result == True: - print("✓ Test 22: Optional json_array attribute missing - passes") - passed += 1 -else: - print( - "✗ Test 22: FAILED - Should pass when optional json_array attribute is missing" - ) - failed += 1 - -# Test 23: Optional with plain_string type - missing attribute -span = { - "data": { - # gen_ai.response.text is NOT present - } -} -schema = {"type": "plain_string", "min_length": 1, "optional": True} -result = attribute_matches(span, "gen_ai.response.text", schema) - -if result == True: - print("✓ Test 23: Optional plain_string attribute missing - passes") - passed += 1 -else: - print( - "✗ Test 23: FAILED - Should pass when optional plain_string attribute is missing" - ) - failed += 1 - -# ============================================================================ -# JSON Array length_lte Constraint Tests -# ============================================================================ - -print("\n--- JSON Array length_lte Constraint Tests ---\n") - -# Test 24: Valid length_lte constraint (array length 2 <= original_length 5) -span = { - "data": { - "gen_ai.request.messages": json.dumps( - [{"role": "system", "content": "Hello"}, {"role": "user", "content": "Hi"}] - ), - "gen_ai.request.messages.original_length": 5, - } -} -schema = {"type": "json_array", "length_lte": "gen_ai.request.messages.original_length"} -result = attribute_matches(span, "gen_ai.request.messages", schema) - -if result == True: - print( - "✓ Test 24: Valid length_lte constraint (array length 2 <= original_length 5)" - ) - passed += 1 -else: - print("✗ Test 24: FAILED - Should pass when array length <= other attribute") - failed += 1 - -# Test 25: Invalid length_lte constraint (array length 3 > original_length 2) -span = { - "data": { - "gen_ai.request.messages": json.dumps( - [ - {"role": "system", "content": "Hello"}, - {"role": "user", "content": "Hi"}, - {"role": "assistant", "content": "Bye"}, - ] - ), - "gen_ai.request.messages.original_length": 2, - } -} -schema = {"type": "json_array", "length_lte": "gen_ai.request.messages.original_length"} -result = attribute_matches(span, "gen_ai.request.messages", schema) - -if result == False: - print( - "✓ Test 25: Invalid length_lte constraint detected (array length 3 > original_length 2)" - ) - passed += 1 -else: - print("✗ Test 25: FAILED - Should fail when array length > other attribute") - failed += 1 - -# Test 26: Edge case - length_lte with equal values (array length 3 <= original_length 3) -span = { - "data": { - "gen_ai.request.messages": json.dumps( - [ - {"role": "system", "content": "Hello"}, - {"role": "user", "content": "Hi"}, - {"role": "assistant", "content": "Bye"}, - ] - ), - "gen_ai.request.messages.original_length": 3, - } -} -schema = {"type": "json_array", "length_lte": "gen_ai.request.messages.original_length"} -result = attribute_matches(span, "gen_ai.request.messages", schema) - -if result == True: - print( - "✓ Test 26: Edge case - length_lte with equal values (array length 3 <= original_length 3)" - ) - passed += 1 -else: - print("✗ Test 26: FAILED - Should pass when array length == other attribute") - failed += 1 - -# Test 27: length_lte passes when other attribute is missing -span = { - "data": { - "gen_ai.request.messages": json.dumps([{"role": "user", "content": "Hi"}]) - # gen_ai.request.messages.original_length is NOT present - } -} -schema = {"type": "json_array", "length_lte": "gen_ai.request.messages.original_length"} -result = attribute_matches(span, "gen_ai.request.messages", schema) - -if result == True: - print("✓ Test 27: length_lte passes when other attribute is missing") - passed += 1 -else: - print("✗ Test 27: FAILED - Should pass when other attribute is missing") - failed += 1 - -# Test 28: length_lte combined with items_have -span = { - "data": { - "gen_ai.request.messages": json.dumps( - [{"role": "system", "content": "Hello"}, {"role": "user", "content": "Hi"}] - ), - "gen_ai.request.messages.original_length": 5, - } -} -schema = { - "type": "json_array", - "length_lte": "gen_ai.request.messages.original_length", - "items_have": ["role", "content"], -} -result = attribute_matches(span, "gen_ai.request.messages", schema) - -if result == True: - print("✓ Test 28: length_lte combined with items_have - passes") - passed += 1 -else: - print("✗ Test 28: FAILED - Should pass with valid length_lte and items_have") - failed += 1 - -# Test 29: length_lte passes but items_have fails -span = { - "data": { - "gen_ai.request.messages": json.dumps( - [ - {"role": "system"}, # Missing 'content' - {"role": "user", "content": "Hi"}, - ] - ), - "gen_ai.request.messages.original_length": 5, - } -} -schema = { - "type": "json_array", - "length_lte": "gen_ai.request.messages.original_length", - "items_have": ["role", "content"], -} -result = attribute_matches(span, "gen_ai.request.messages", schema) - -if result == False: - print("✓ Test 29: length_lte passes but items_have fails - overall fails") - passed += 1 -else: - print("✗ Test 29: FAILED - Should fail when items_have validation fails") - failed += 1 - -# ============================================================================ -# JSON Array contains Constraint Tests -# ============================================================================ - -print("\n--- JSON Array contains Constraint Tests ---\n") - -# Test 30: Valid contains - JSON string contains "[Blob substitute]" -span = { - "data": { - "gen_ai.request.messages": json.dumps( - [ - { - "role": "user", - "content": [ - {"type": "text", "text": "Describe this image"}, - {"type": "image", "data": "[Blob substitute]"}, - ], - } - ] - ) - } -} -schema = {"type": "json_array", "contains": "[Blob substitute]"} -result = attribute_matches(span, "gen_ai.request.messages", schema) - -if result == True: - print('✓ Test 30: Valid contains - JSON string contains "[Blob substitute]"') - passed += 1 -else: - print("✗ Test 30: FAILED - Should pass when JSON contains the substring") - failed += 1 - -# Test 31: Invalid contains - JSON string does not contain "[Blob substitute]" -span = { - "data": { - "gen_ai.request.messages": json.dumps( - [{"role": "user", "content": "Hello world"}] - ) - } -} -schema = {"type": "json_array", "contains": "[Blob substitute]"} -result = attribute_matches(span, "gen_ai.request.messages", schema) - -if result == False: - print( - '✓ Test 31: Invalid contains - JSON string does not contain "[Blob substitute]"' - ) - passed += 1 -else: - print("✗ Test 31: FAILED - Should fail when JSON does not contain the substring") - failed += 1 - -# Test 32: contains combined with min_length -span = { - "data": { - "gen_ai.request.messages": json.dumps( - [ - {"role": "system", "content": "You are helpful"}, - {"role": "user", "content": "[Blob substitute]"}, - ] - ) - } -} -schema = {"type": "json_array", "min_length": 2, "contains": "[Blob substitute]"} -result = attribute_matches(span, "gen_ai.request.messages", schema) - -if result == True: - print("✓ Test 32: contains combined with min_length - passes") - passed += 1 -else: - print("✗ Test 32: FAILED - Should pass with valid contains and min_length") - failed += 1 - -# Test 33: contains passes but min_length fails -span = { - "data": { - "gen_ai.request.messages": json.dumps( - [{"role": "user", "content": "[Blob substitute]"}] - ) - } -} -schema = {"type": "json_array", "min_length": 3, "contains": "[Blob substitute]"} -result = attribute_matches(span, "gen_ai.request.messages", schema) - -if result == False: - print("✓ Test 33: contains passes but min_length fails - overall fails") - passed += 1 -else: - print("✗ Test 33: FAILED - Should fail when min_length validation fails") - failed += 1 - -print(f"\n{passed} passed, {failed} failed") -sys.exit(1 if failed > 0 else 0) diff --git a/sdks/py/anthropic/cases/1-simple.py b/sdks/py/anthropic/cases/1-simple.py deleted file mode 100644 index ce8de21..0000000 --- a/sdks/py/anthropic/cases/1-simple.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -1-simple: Basic Completion - -Tests a simple chat completion request with Anthropic SDK -and verifies that Sentry captures the appropriate spans and AI monitoring data. -""" - -import os -from anthropic import Anthropic -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - prompt = inputs["prompt"] - - client = Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY")) - - response = client.messages.create( - model=model, - system=system, - messages=[ - {"role": "user", "content": prompt} - ], - max_tokens=1024, - ) - - text = response.content[0].text - - if not text: - raise Exception("No output returned from Anthropic") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text}") - - return text - - -# Export test case functions -test_case = run_test_case("1-simple", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/anthropic/cases/10-binary-content-redaction.py b/sdks/py/anthropic/cases/10-binary-content-redaction.py deleted file mode 100644 index 32ee7c4..0000000 --- a/sdks/py/anthropic/cases/10-binary-content-redaction.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -10-binary-content-redaction: Binary Content Redaction Test - -Tests that when binary data (such as images) is sent to an LLM, Sentry -correctly redacts the binary content in the captured span data and -replaces it with a substitute marker. -""" - -import os -import base64 -from pathlib import Path -from anthropic import Anthropic -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - - client = Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY")) - - # Load static test image - # Path: cases -> anthropic -> py -> sdks -> repo_root - repo_root = Path(__file__).parent.parent.parent.parent.parent - image_path = repo_root / "shared" / "test-assets" / "test-image-10x10-red.png" - - with open(image_path, "rb") as f: - image_data = f.read() - base64_image = base64.standard_b64encode(image_data).decode("utf-8") - - # Send message with image content - response = client.messages.create( - model=model, - messages=[ - { - "role": "user", - "content": [ - { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": base64_image, - }, - }, - { - "type": "text", - "text": "What color is this image? Answer in one word.", - }, - ], - } - ], - max_tokens=1024, - ) - - text = response.content[0].text - - if not text: - raise Exception("No output returned from Anthropic") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text}") - print(f" Sent image with {len(image_data)} bytes of binary data") - - return text - - -# Export test case functions -test_case = run_test_case("10-binary-content-redaction", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/anthropic/cases/2-multi-step.py b/sdks/py/anthropic/cases/2-multi-step.py deleted file mode 100644 index 2e0299a..0000000 --- a/sdks/py/anthropic/cases/2-multi-step.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -2-multi-step: Multi-step Conversation - -Tests a multi-step conversation with conversation history using Anthropic SDK -and verifies that Sentry captures all spans for both API calls. -""" - -import os -from anthropic import Anthropic -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - first_prompt = inputs["first_prompt"] - second_prompt = inputs["second_prompt"] - - client = Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY")) - - # First call - first_response = client.messages.create( - model=model, - system=system, - messages=[ - {"role": "user", "content": first_prompt} - ], - max_tokens=1024, - ) - - first_text = first_response.content[0].text - - if not first_text: - raise Exception("No output returned from Anthropic (first call)") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" First response: {first_text}") - - # Second call with conversation history - second_response = client.messages.create( - model=model, - system=system, - messages=[ - {"role": "user", "content": first_prompt}, - {"role": "assistant", "content": first_text}, - {"role": "user", "content": second_prompt} - ], - max_tokens=1024, - ) - - second_text = second_response.content[0].text - - if not second_text: - raise Exception("No output returned from Anthropic (second call)") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Second response: {second_text}") - - return second_text - - -# Export test case functions -test_case = run_test_case("2-multi-step", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/anthropic/cases/9-message-truncation.py b/sdks/py/anthropic/cases/9-message-truncation.py deleted file mode 100644 index 5f4bf44..0000000 --- a/sdks/py/anthropic/cases/9-message-truncation.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -9-message-truncation: Message Truncation Test - -Tests that when large messages are sent to an LLM, Sentry correctly tracks -the original message count vs. the potentially truncated message count in -the captured span data. -""" - -import os -from anthropic import Anthropic -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - message_size_kb = inputs.get("message_size_kb", 9) - message_count = inputs.get("message_count", 3) - - client = Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY")) - - # Generate large content for each message (~9KB each) - large_content = "x" * (message_size_kb * 1024) - - # Create the messages array with large content - messages = [] - for i in range(message_count): - role = "user" if i % 2 == 0 else "assistant" - messages.append({"role": role, "content": f"Message {i + 1}: {large_content}"}) - - # Ensure the last message is from user (required by Anthropic) - if messages[-1]["role"] == "assistant": - messages.append( - {"role": "user", "content": "Please summarize what we discussed."} - ) - - response = client.messages.create( - model=model, - messages=messages, - max_tokens=1024, - ) - - text = response.content[0].text - - if not text: - raise Exception("No output returned from Anthropic") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text[:100]}...") - print( - f" Sent {len(messages)} messages with ~{message_size_kb}KB content each" - ) - - return text - - -# Export test case functions -test_case = run_test_case("9-message-truncation", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/anthropic/config.json b/sdks/py/anthropic/config.json deleted file mode 100644 index 133a556..0000000 --- a/sdks/py/anthropic/config.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "sdk_name": "anthropic", - "framework_type": "low-level", - "overrides": { - "1-simple": { - "model": "claude-haiku-4-5", - "gen_ai.request.model": "claude-haiku-4-5", - "gen_ai.response.model": "claude-haiku-4-5*" - }, - "2-multi-step": { - "model": "claude-haiku-4-5", - "gen_ai.request.model": "claude-haiku-4-5", - "gen_ai.response.model": "claude-haiku-4-5*" - }, - "9-message-truncation": { - "model": "claude-haiku-4-5", - "gen_ai.request.model": "claude-haiku-4-5", - "gen_ai.response.model": "claude-haiku-4-5*" - }, - "10-binary-content-redaction": { - "model": "claude-haiku-4-5", - "gen_ai.request.model": "claude-haiku-4-5", - "gen_ai.response.model": "claude-haiku-4-5*" - } - } -} diff --git a/sdks/py/anthropic/requirements.txt b/sdks/py/anthropic/requirements.txt deleted file mode 100644 index 85597f3..0000000 --- a/sdks/py/anthropic/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -sentry-sdk==2.46.0 -anthropic==0.75.0 -python-dotenv==1.2.1 diff --git a/sdks/py/anthropic/setup.py b/sdks/py/anthropic/setup.py deleted file mode 100644 index b59e637..0000000 --- a/sdks/py/anthropic/setup.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -Setup file for Anthropic SDK tests - -Initializes Sentry with Anthropic-specific integrations. -""" - -import os -import sys -from pathlib import Path -from dotenv import load_dotenv -import sentry_sdk - -# Add test utils to path -test_utils_path = Path(__file__).parent.parent / "_test-utils" -sys.path.insert(0, str(test_utils_path)) - -from mock_transport import create_mock_transport, MockTransportCapture -import mock_transport as mt - -# Load environment variables -env_path = Path(__file__).parent.parent.parent.parent / ".env" -load_dotenv(dotenv_path=env_path) - -# Pre-initialize mock transport (required for Python) -mt._mock_transport_capture = MockTransportCapture() - -mock_transport_instance = create_mock_transport() - -# Initialize Sentry with Anthropic integration (auto-enabled) -sentry_sdk.init( - traces_sample_rate=1.0, - transport=mock_transport_instance, - send_default_pii=True, - # AnthropicIntegration is enabled automatically -) diff --git a/sdks/py/google-genai/cases/1-simple.py b/sdks/py/google-genai/cases/1-simple.py deleted file mode 100644 index 50ba844..0000000 --- a/sdks/py/google-genai/cases/1-simple.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -1-simple: Basic Completion - -Tests a simple chat completion request with Google GenAI SDK -and verifies that Sentry captures the appropriate spans and AI monitoring data. -""" - -import os -from google import genai -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - prompt = inputs["prompt"] - - client = genai.Client(api_key=os.getenv("GOOGLE_GENAI_API_KEY")) - - response = client.models.generate_content( - model=model, - contents=prompt, - config=genai.types.GenerateContentConfig( - system_instruction=system, - ), - ) - - if not response.text: - raise Exception("No output returned from Google GenAI") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {response.text}") - - return response.text - - -# Export test case functions -test_case = run_test_case("1-simple", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/google-genai/cases/10-binary-content-redaction.py b/sdks/py/google-genai/cases/10-binary-content-redaction.py deleted file mode 100644 index 6df8109..0000000 --- a/sdks/py/google-genai/cases/10-binary-content-redaction.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -10-binary-content-redaction: Binary Content Redaction Test - -Tests that when binary data (such as images) is sent to an LLM, Sentry -correctly redacts the binary content in the captured span data and -replaces it with a substitute marker. -""" - -import os -from pathlib import Path -from google import genai -from google.genai import types -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - - client = genai.Client(api_key=os.getenv("GOOGLE_GENAI_API_KEY")) - - # Load static test image - # Path: cases -> google-genai -> py -> sdks -> repo_root - repo_root = Path(__file__).parent.parent.parent.parent.parent - image_path = repo_root / "shared" / "test-assets" / "test-image-10x10-red.png" - - with open(image_path, "rb") as f: - image_data = f.read() - - # Google GenAI accepts images as Part objects - image_part = types.Part.from_bytes(data=image_data, mime_type="image/png") - text_part = types.Part(text="What color is this image? Answer in one word.") - - response = client.models.generate_content( - model=model, - contents=[image_part, text_part], - ) - - if not response.text: - raise Exception("No output returned from Google GenAI") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {response.text}") - print(f" Sent image with {len(image_data)} bytes of binary data") - - return response.text - - -# Export test case functions -test_case = run_test_case("10-binary-content-redaction", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/google-genai/cases/2-multi-step.py b/sdks/py/google-genai/cases/2-multi-step.py deleted file mode 100644 index 1f65939..0000000 --- a/sdks/py/google-genai/cases/2-multi-step.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -2-multi-step: Multi-step Conversation - -Tests a multi-step conversation with conversation history using Google GenAI SDK -and verifies that Sentry captures all spans for both API calls. -""" - -import os -from google import genai -from google.genai import types -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - first_prompt = inputs["first_prompt"] - second_prompt = inputs["second_prompt"] - - client = genai.Client(api_key=os.getenv("GOOGLE_GENAI_API_KEY")) - - # First call - first_response = client.models.generate_content( - model=model, - contents=first_prompt, - config=types.GenerateContentConfig( - system_instruction=system, - ), - ) - - first_text = first_response.text - - if not first_text: - raise Exception("No output returned from Google GenAI (first call)") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" First response: {first_text}") - - # Second call with conversation history - # Google GenAI API expects contents as a list for multi-turn conversations - second_response = client.models.generate_content( - model=model, - contents=[ - types.Content( - role="user", - parts=[types.Part(text=first_prompt)] - ), - types.Content( - role="model", - parts=[types.Part(text=first_text)] - ), - types.Content( - role="user", - parts=[types.Part(text=second_prompt)] - ), - ], - config=types.GenerateContentConfig( - system_instruction=system, - ), - ) - - second_text = second_response.text - - if not second_text: - raise Exception("No output returned from Google GenAI (second call)") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Second response: {second_text}") - - return second_text - - -# Export test case functions -test_case = run_test_case("2-multi-step", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/google-genai/cases/9-message-truncation.py b/sdks/py/google-genai/cases/9-message-truncation.py deleted file mode 100644 index 982eaab..0000000 --- a/sdks/py/google-genai/cases/9-message-truncation.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -9-message-truncation: Message Truncation Test - -Tests that when large messages are sent to an LLM, Sentry correctly tracks -the original message count vs. the potentially truncated message count in -the captured span data. -""" - -import os -from google import genai -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - message_size_kb = inputs.get("message_size_kb", 9) - message_count = inputs.get("message_count", 3) - - client = genai.Client(api_key=os.getenv("GOOGLE_GENAI_API_KEY")) - - # Generate large content for each message (~9KB each) - large_content = "x" * (message_size_kb * 1024) - - # Build a large prompt with multiple "messages" embedded - prompt_parts = [] - for i in range(message_count): - prompt_parts.append(f"Message {i + 1}: {large_content}") - - large_prompt = ( - "\n\n".join(prompt_parts) + "\n\nPlease summarize the above messages briefly." - ) - - response = client.models.generate_content( - model=model, - contents=large_prompt, - ) - - if not response.text: - raise Exception("No output returned from Google GenAI") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {response.text[:100]}...") - print( - f" Sent {message_count} messages with ~{message_size_kb}KB content each" - ) - - return response.text - - -# Export test case functions -test_case = run_test_case("9-message-truncation", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/google-genai/config.json b/sdks/py/google-genai/config.json deleted file mode 100644 index 8b8395e..0000000 --- a/sdks/py/google-genai/config.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "sdk_name": "google-genai", - "framework_type": "low-level", - "overrides": { - "1-simple": { - "model": "gemini-2.5-flash-lite", - "gen_ai.request.model": "gemini-2.5-flash-lite", - "gen_ai.response.model": "gemini-2.5-flash-lite" - }, - "2-multi-step": { - "model": "gemini-2.5-flash-lite", - "gen_ai.request.model": "gemini-2.5-flash-lite", - "gen_ai.response.model": "gemini-2.5-flash-lite" - }, - "9-message-truncation": { - "model": "gemini-2.5-flash-lite", - "gen_ai.request.model": "gemini-2.5-flash-lite", - "gen_ai.response.model": "gemini-2.5-flash-lite" - }, - "10-binary-content-redaction": { - "model": "gemini-2.5-flash-lite", - "gen_ai.request.model": "gemini-2.5-flash-lite", - "gen_ai.response.model": "gemini-2.5-flash-lite" - } - } -} diff --git a/sdks/py/google-genai/requirements.txt b/sdks/py/google-genai/requirements.txt deleted file mode 100644 index 0e35238..0000000 --- a/sdks/py/google-genai/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -sentry-sdk==2.46.0 -google-genai==1.49.0 -python-dotenv==1.2.1 diff --git a/sdks/py/google-genai/setup.py b/sdks/py/google-genai/setup.py deleted file mode 100644 index b744657..0000000 --- a/sdks/py/google-genai/setup.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Setup file for Google GenAI SDK tests - -Initializes Sentry with Google GenAI-specific integrations. -""" - -import os -import sys -from pathlib import Path -from dotenv import load_dotenv -from sentry_sdk.integrations.google_genai import GoogleGenAIIntegration -import sentry_sdk - -# Add test utils to path -test_utils_path = Path(__file__).parent.parent / "_test-utils" -sys.path.insert(0, str(test_utils_path)) - -from mock_transport import create_mock_transport, MockTransportCapture -import mock_transport as mt - -# Load environment variables -env_path = Path(__file__).parent.parent.parent.parent / ".env" -load_dotenv(dotenv_path=env_path) - -# Pre-initialize mock transport (required for Python) -mt._mock_transport_capture = MockTransportCapture() - -mock_transport_instance = create_mock_transport() - -# Initialize Sentry with Google GenAI integration -sentry_sdk.init( - traces_sample_rate=1.0, - transport=mock_transport_instance, - send_default_pii=True, - integrations=[GoogleGenAIIntegration(include_prompts=True)], -) diff --git a/sdks/py/langchain/cases/1-simple.py b/sdks/py/langchain/cases/1-simple.py deleted file mode 100644 index dd7960c..0000000 --- a/sdks/py/langchain/cases/1-simple.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -1-simple: Basic Completion - -Tests a simple chat completion request with LangChain SDK -and verifies that Sentry captures the appropriate spans and AI monitoring data. -""" - -import os -from langchain_openai import ChatOpenAI -from langchain_core.messages import HumanMessage, SystemMessage -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - prompt = inputs["prompt"] - - chat = ChatOpenAI(model=model, api_key=os.getenv("OPENAI_API_KEY")) - - messages = [ - SystemMessage(content=system), - HumanMessage(content=prompt), - ] - - response = chat.invoke(messages) - - text = response.content - - if not text: - raise Exception("No output returned from LangChain") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text}") - - return text - - -# Export test case functions -test_case = run_test_case("1-simple", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/langchain/cases/10-binary-content-redaction.py b/sdks/py/langchain/cases/10-binary-content-redaction.py deleted file mode 100644 index a9a8b6d..0000000 --- a/sdks/py/langchain/cases/10-binary-content-redaction.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -10-binary-content-redaction: Binary Content Redaction Test - -Tests that when binary data (such as images) is sent to an LLM, Sentry -correctly redacts the binary content in the captured span data and -replaces it with a substitute marker. -""" - -import os -import base64 -from pathlib import Path -from langchain_openai import ChatOpenAI -from langchain_core.messages import HumanMessage -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - - chat = ChatOpenAI(model=model, api_key=os.getenv("OPENAI_API_KEY")) - - # Load static test image - # Path: cases -> langchain -> py -> sdks -> repo_root - repo_root = Path(__file__).parent.parent.parent.parent.parent - image_path = repo_root / "shared" / "test-assets" / "test-image-10x10-red.png" - - with open(image_path, "rb") as f: - image_data = f.read() - base64_image = base64.standard_b64encode(image_data).decode("utf-8") - - # LangChain uses a specific format for image inputs - message = HumanMessage( - content=[ - { - "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{base64_image}"}, - }, - { - "type": "text", - "text": "What color is this image? Answer in one word.", - }, - ] - ) - - response = chat.invoke([message]) - - text = response.content - - if not text: - raise Exception("No output returned from LangChain") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text}") - print(f" Sent image with {len(image_data)} bytes of binary data") - - return text - - -# Export test case functions -test_case = run_test_case("10-binary-content-redaction", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/langchain/cases/2-multi-step.py b/sdks/py/langchain/cases/2-multi-step.py deleted file mode 100644 index 6de53b2..0000000 --- a/sdks/py/langchain/cases/2-multi-step.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -2-multi-step: Multi-step Conversation - -Tests a multi-step conversation with conversation history using LangChain SDK -and verifies that Sentry captures all spans for both API calls. -""" - -import os -from langchain_openai import ChatOpenAI -from langchain_core.messages import HumanMessage, SystemMessage, AIMessage -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - first_prompt = inputs["first_prompt"] - second_prompt = inputs["second_prompt"] - - chat = ChatOpenAI(model=model, api_key=os.getenv("OPENAI_API_KEY")) - - # First call - first_messages = [ - SystemMessage(content=system), - HumanMessage(content=first_prompt), - ] - - first_response = chat.invoke(first_messages) - first_text = first_response.content - - if not first_text: - raise Exception("No output returned from LangChain (first call)") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" First response: {first_text}") - - # Second call with conversation history - second_messages = [ - SystemMessage(content=system), - HumanMessage(content=first_prompt), - AIMessage(content=first_text), - HumanMessage(content=second_prompt), - ] - - second_response = chat.invoke(second_messages) - second_text = second_response.content - - if not second_text: - raise Exception("No output returned from LangChain (second call)") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Second response: {second_text}") - - return second_text - - -# Export test case functions -test_case = run_test_case("2-multi-step", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/langchain/cases/9-message-truncation.py b/sdks/py/langchain/cases/9-message-truncation.py deleted file mode 100644 index 9485dd4..0000000 --- a/sdks/py/langchain/cases/9-message-truncation.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -9-message-truncation: Message Truncation Test - -Tests that when large messages are sent to an LLM, Sentry correctly tracks -the original message count vs. the potentially truncated message count in -the captured span data. -""" - -import os -from langchain_openai import ChatOpenAI -from langchain_core.messages import HumanMessage, SystemMessage, AIMessage -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - message_size_kb = inputs.get("message_size_kb", 9) - message_count = inputs.get("message_count", 3) - - chat = ChatOpenAI(model=model, api_key=os.getenv("OPENAI_API_KEY")) - - # Generate large content for each message (~9KB each) - large_content = "x" * (message_size_kb * 1024) - - # Create the messages array with large content - messages = [] - for i in range(message_count): - if i % 2 == 0: - messages.append(HumanMessage(content=f"Message {i + 1}: {large_content}")) - else: - messages.append(AIMessage(content=f"Message {i + 1}: {large_content}")) - - # Ensure the last message is from user (required) - if isinstance(messages[-1], AIMessage): - messages.append(HumanMessage(content="Please summarize what we discussed.")) - - response = chat.invoke(messages) - - text = response.content - - if not text: - raise Exception("No output returned from LangChain") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text[:100]}...") - print( - f" Sent {len(messages)} messages with ~{message_size_kb}KB content each" - ) - - return text - - -# Export test case functions -test_case = run_test_case("9-message-truncation", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/langchain/config.json b/sdks/py/langchain/config.json deleted file mode 100644 index f634cbb..0000000 --- a/sdks/py/langchain/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "sdk_name": "langchain", - "framework_type": "low-level", - "overrides": {} -} diff --git a/sdks/py/langchain/requirements.txt b/sdks/py/langchain/requirements.txt deleted file mode 100644 index 70c978d..0000000 --- a/sdks/py/langchain/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -sentry-sdk==2.46.0 -langchain-core==1.0.4 -langchain-openai==1.0.2 -python-dotenv==1.2.1 diff --git a/sdks/py/langchain/setup.py b/sdks/py/langchain/setup.py deleted file mode 100644 index 4691766..0000000 --- a/sdks/py/langchain/setup.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -Setup file for LangChain SDK tests - -Initializes Sentry with LangChain-specific integrations. -""" - -import os -import sys -from pathlib import Path -from dotenv import load_dotenv -import sentry_sdk - -# Add test utils to path -test_utils_path = Path(__file__).parent.parent / "_test-utils" -sys.path.insert(0, str(test_utils_path)) - -from mock_transport import create_mock_transport, MockTransportCapture -import mock_transport as mt - -# Load environment variables -env_path = Path(__file__).parent.parent.parent.parent / ".env" -load_dotenv(dotenv_path=env_path) - -# Pre-initialize mock transport (required for Python) -mt._mock_transport_capture = MockTransportCapture() - -mock_transport_instance = create_mock_transport() - -# Initialize Sentry with LangChain integration (auto-enabled) -sentry_sdk.init( - traces_sample_rate=1.0, - transport=mock_transport_instance, - send_default_pii=True, - # LangChainIntegration is enabled automatically -) diff --git a/sdks/py/langgraph/cases/1-simple.py b/sdks/py/langgraph/cases/1-simple.py deleted file mode 100644 index 06d386e..0000000 --- a/sdks/py/langgraph/cases/1-simple.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -1-simple: Basic Completion - -Tests a simple chat completion request with LangGraph SDK -and verifies that Sentry captures the appropriate spans and AI monitoring data. -""" - -import os -from langgraph.prebuilt import create_react_agent -from langchain_openai import ChatOpenAI -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - prompt = inputs["prompt"] - - llm = ChatOpenAI(model=model, api_key=os.getenv("OPENAI_API_KEY")) - - # Create a simple react agent with no tools - agent = create_react_agent(llm, tools=[]) - - # Combine system and prompt as LangGraph expects - messages = [ - ("system", system), - ("user", prompt) - ] - - result = agent.invoke({"messages": messages}) - - # Extract the AI's response from the result - text = result["messages"][-1].content - - if not text: - raise Exception("No output returned from LangGraph") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text}") - - return text - - -# Export test case functions -test_case = run_test_case("1-simple", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/langgraph/cases/10-binary-content-redaction.py b/sdks/py/langgraph/cases/10-binary-content-redaction.py deleted file mode 100644 index 8053289..0000000 --- a/sdks/py/langgraph/cases/10-binary-content-redaction.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -10-binary-content-redaction: Binary Content Redaction Test - -Tests that when binary data (such as images) is sent to an LLM via LangGraph, -Sentry correctly redacts the binary content in the captured span data and -replaces it with a substitute marker. -""" - -import os -import base64 -from pathlib import Path -from langgraph.prebuilt import create_react_agent -from langchain_openai import ChatOpenAI -from langchain_core.messages import HumanMessage -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - - llm = ChatOpenAI(model=model, api_key=os.getenv("OPENAI_API_KEY")) - - # Create a simple react agent with no tools - agent = create_react_agent(llm, tools=[]) - - # Load static test image - # Path: cases -> langgraph -> py -> sdks -> repo_root - repo_root = Path(__file__).parent.parent.parent.parent.parent - image_path = repo_root / "shared" / "test-assets" / "test-image-10x10-red.png" - - with open(image_path, "rb") as f: - image_data = f.read() - base64_image = base64.standard_b64encode(image_data).decode("utf-8") - - # Create message with image content - message = HumanMessage( - content=[ - { - "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{base64_image}"}, - }, - { - "type": "text", - "text": "What color is this image? Answer in one word.", - }, - ] - ) - - result = agent.invoke({"messages": [message]}) - - # Extract the AI's response from the result - text = result["messages"][-1].content - - if not text: - raise Exception("No output returned from LangGraph") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text}") - print(f" Sent image with {len(image_data)} bytes of binary data") - - return text - - -# Export test case functions -test_case = run_test_case("10-binary-content-redaction", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/langgraph/cases/2-multi-step.py b/sdks/py/langgraph/cases/2-multi-step.py deleted file mode 100644 index f8adb2f..0000000 --- a/sdks/py/langgraph/cases/2-multi-step.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -2-multi-step: Multi-step Conversation - -Tests a multi-step conversation with conversation history using LangGraph SDK -and verifies that Sentry captures all spans for both API calls. -""" - -import os -from langgraph.prebuilt import create_react_agent -from langchain_openai import ChatOpenAI -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - first_prompt = inputs["first_prompt"] - second_prompt = inputs["second_prompt"] - - llm = ChatOpenAI(model=model, api_key=os.getenv("OPENAI_API_KEY")) - - # Create a simple react agent with no tools - agent = create_react_agent(llm, tools=[]) - - # First call - first_messages = [ - ("system", system), - ("user", first_prompt) - ] - - first_result = agent.invoke({"messages": first_messages}) - first_text = first_result["messages"][-1].content - - if not first_text: - raise Exception("No output returned from LangGraph (first call)") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" First response: {first_text}") - - # Second call with conversation history - second_messages = [ - ("system", system), - ("user", first_prompt), - ("ai", first_text), - ("user", second_prompt) - ] - - second_result = agent.invoke({"messages": second_messages}) - second_text = second_result["messages"][-1].content - - if not second_text: - raise Exception("No output returned from LangGraph (second call)") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Second response: {second_text}") - - return second_text - - -# Export test case functions -test_case = run_test_case("2-multi-step", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/langgraph/cases/9-message-truncation.py b/sdks/py/langgraph/cases/9-message-truncation.py deleted file mode 100644 index 4eb855d..0000000 --- a/sdks/py/langgraph/cases/9-message-truncation.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -9-message-truncation: Message Truncation Test - -Tests that when large messages are sent to an LLM via LangGraph, Sentry -correctly tracks the original message count vs. the potentially truncated -message count in the captured span data. -""" - -import os -from langgraph.prebuilt import create_react_agent -from langchain_openai import ChatOpenAI -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - message_size_kb = inputs.get("message_size_kb", 9) - message_count = inputs.get("message_count", 3) - - llm = ChatOpenAI(model=model, api_key=os.getenv("OPENAI_API_KEY")) - - # Create a simple react agent with no tools - agent = create_react_agent(llm, tools=[]) - - # Generate large content for each message (~9KB each) - large_content = "x" * (message_size_kb * 1024) - - # Build messages array with large content - messages = [] - for i in range(message_count): - role = "user" if i % 2 == 0 else "assistant" - messages.append((role, f"Message {i + 1}: {large_content}")) - - # Ensure the last message is from user - if messages[-1][0] == "assistant": - messages.append(("user", "Please summarize what we discussed.")) - - result = agent.invoke({"messages": messages}) - - # Extract the AI's response from the result - text = result["messages"][-1].content - - if not text: - raise Exception("No output returned from LangGraph") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text[:100]}...") - print( - f" Sent {len(messages)} messages with ~{message_size_kb}KB content each" - ) - - return text - - -# Export test case functions -test_case = run_test_case("9-message-truncation", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/langgraph/config.json b/sdks/py/langgraph/config.json deleted file mode 100644 index 0ae5311..0000000 --- a/sdks/py/langgraph/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "sdk_name": "langgraph", - "framework_type": "agentic", - "overrides": {} -} diff --git a/sdks/py/langgraph/requirements.txt b/sdks/py/langgraph/requirements.txt deleted file mode 100644 index 35b098b..0000000 --- a/sdks/py/langgraph/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -sentry-sdk==2.46.0 -langgraph==1.0.3 -langchain-openai>=1.0.0 -python-dotenv==1.2.1 diff --git a/sdks/py/langgraph/setup.py b/sdks/py/langgraph/setup.py deleted file mode 100644 index b567a5f..0000000 --- a/sdks/py/langgraph/setup.py +++ /dev/null @@ -1,38 +0,0 @@ -""" -Setup file for LangGraph SDK tests - -Initializes Sentry with LangGraph-specific integrations. -""" - -import os -import sys -from pathlib import Path -from dotenv import load_dotenv -from sentry_sdk.integrations.openai import OpenAIIntegration -import sentry_sdk - -# Add test utils to path -test_utils_path = Path(__file__).parent.parent / "_test-utils" -sys.path.insert(0, str(test_utils_path)) - -from mock_transport import create_mock_transport, MockTransportCapture -import mock_transport as mt - -# Load environment variables -env_path = Path(__file__).parent.parent.parent.parent / ".env" -load_dotenv(dotenv_path=env_path) - -# Pre-initialize mock transport (required for Python) -mt._mock_transport_capture = MockTransportCapture() - -mock_transport_instance = create_mock_transport() - -# Initialize Sentry with LangGraph integration (auto-enabled) -# Disable OpenAI integration to avoid double counting -sentry_sdk.init( - traces_sample_rate=1.0, - transport=mock_transport_instance, - send_default_pii=True, - disabled_integrations=[OpenAIIntegration()], - # LanggraphIntegration is enabled automatically -) diff --git a/sdks/py/litellm/cases/1-simple.py b/sdks/py/litellm/cases/1-simple.py deleted file mode 100644 index 086bb7a..0000000 --- a/sdks/py/litellm/cases/1-simple.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -1-simple: Basic Completion - -Tests a simple chat completion request with LiteLLM SDK -and verifies that Sentry captures the appropriate spans and AI monitoring data. -""" - -import os -from litellm import completion -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - prompt = inputs["prompt"] - - # LiteLLM requires the model prefix (e.g., "openai/gpt-5-nano") - response = completion( - model=f"openai/{model}", - messages=[ - {"role": "system", "content": system}, - {"role": "user", "content": prompt} - ] - ) - - text = response.choices[0].message.content - - if not text: - raise Exception("No output returned from LiteLLM") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text}") - - return text - - -# Export test case functions -test_case = run_test_case("1-simple", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/litellm/cases/10-binary-content-redaction.py b/sdks/py/litellm/cases/10-binary-content-redaction.py deleted file mode 100644 index b62b7d8..0000000 --- a/sdks/py/litellm/cases/10-binary-content-redaction.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -10-binary-content-redaction: Binary Content Redaction Test - -Tests that when binary data (such as images) is sent to an LLM via LiteLLM, -Sentry correctly redacts the binary content in the captured span data and -replaces it with a substitute marker. -""" - -import os -import base64 -from pathlib import Path -from litellm import completion -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - - # Load static test image - # Path: cases -> litellm -> py -> sdks -> repo_root - repo_root = Path(__file__).parent.parent.parent.parent.parent - image_path = repo_root / "shared" / "test-assets" / "test-image-10x10-red.png" - - with open(image_path, "rb") as f: - image_data = f.read() - base64_image = base64.standard_b64encode(image_data).decode("utf-8") - - # LiteLLM uses OpenAI-compatible format for vision - # LiteLLM requires the model prefix (e.g., "openai/gpt-5-nano") - response = completion( - model=f"openai/{model}", - messages=[ - { - "role": "user", - "content": [ - { - "type": "image_url", - "image_url": { - "url": f"data:image/png;base64,{base64_image}", - }, - }, - { - "type": "text", - "text": "What color is this image? Answer in one word.", - }, - ], - } - ], - ) - - text = response.choices[0].message.content - - if not text: - raise Exception("No output returned from LiteLLM") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text}") - print(f" Sent image with {len(image_data)} bytes of binary data") - - return text - - -# Export test case functions -test_case = run_test_case("10-binary-content-redaction", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/litellm/cases/2-multi-step.py b/sdks/py/litellm/cases/2-multi-step.py deleted file mode 100644 index 5cfc17c..0000000 --- a/sdks/py/litellm/cases/2-multi-step.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -2-multi-step: Multi-step Conversation - -Tests a multi-step conversation with conversation history using LiteLLM SDK -and verifies that Sentry captures all spans for both API calls. -""" - -import os -from litellm import completion -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - first_prompt = inputs["first_prompt"] - second_prompt = inputs["second_prompt"] - - # First call - first_response = completion( - model=f"openai/{model}", - messages=[ - {"role": "system", "content": system}, - {"role": "user", "content": first_prompt} - ] - ) - - first_text = first_response.choices[0].message.content - - if not first_text: - raise Exception("No output returned from LiteLLM (first call)") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" First response: {first_text}") - - # Second call with conversation history - second_response = completion( - model=f"openai/{model}", - messages=[ - {"role": "system", "content": system}, - {"role": "user", "content": first_prompt}, - {"role": "assistant", "content": first_text}, - {"role": "user", "content": second_prompt} - ] - ) - - second_text = second_response.choices[0].message.content - - if not second_text: - raise Exception("No output returned from LiteLLM (second call)") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Second response: {second_text}") - - return second_text - - -# Export test case functions -test_case = run_test_case("2-multi-step", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/litellm/cases/9-message-truncation.py b/sdks/py/litellm/cases/9-message-truncation.py deleted file mode 100644 index b99acc1..0000000 --- a/sdks/py/litellm/cases/9-message-truncation.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -9-message-truncation: Message Truncation Test - -Tests that when large messages are sent to an LLM via LiteLLM, Sentry -correctly tracks the original message count vs. the potentially truncated -message count in the captured span data. -""" - -import os -from litellm import completion -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - message_size_kb = inputs.get("message_size_kb", 9) - message_count = inputs.get("message_count", 3) - - # Generate large content for each message (~9KB each) - large_content = "x" * (message_size_kb * 1024) - - # Create the messages array with large content - messages = [] - for i in range(message_count): - role = "user" if i % 2 == 0 else "assistant" - messages.append({"role": role, "content": f"Message {i + 1}: {large_content}"}) - - # Ensure the last message is from user (required) - if messages[-1]["role"] == "assistant": - messages.append( - {"role": "user", "content": "Please summarize what we discussed."} - ) - - # LiteLLM requires the model prefix (e.g., "openai/gpt-5-nano") - response = completion( - model=f"openai/{model}", - messages=messages, - ) - - text = response.choices[0].message.content - - if not text: - raise Exception("No output returned from LiteLLM") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text[:100]}...") - print( - f" Sent {len(messages)} messages with ~{message_size_kb}KB content each" - ) - - return text - - -# Export test case functions -test_case = run_test_case("9-message-truncation", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/litellm/config.json b/sdks/py/litellm/config.json deleted file mode 100644 index 0cc3a28..0000000 --- a/sdks/py/litellm/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "sdk_name": "litellm", - "framework_type": "low-level", - "overrides": {} -} diff --git a/sdks/py/litellm/requirements.txt b/sdks/py/litellm/requirements.txt deleted file mode 100644 index 80fcf4a..0000000 --- a/sdks/py/litellm/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -sentry-sdk==2.46.0 -litellm==1.79.3 -python-dotenv==1.2.1 diff --git a/sdks/py/litellm/setup.py b/sdks/py/litellm/setup.py deleted file mode 100644 index cd87858..0000000 --- a/sdks/py/litellm/setup.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Setup file for LiteLLM SDK tests - -Initializes Sentry with LiteLLM-specific integrations. -""" - -import os -import sys -from pathlib import Path -from dotenv import load_dotenv -from sentry_sdk.integrations.litellm import LiteLLMIntegration -import sentry_sdk - -# Add test utils to path -test_utils_path = Path(__file__).parent.parent / "_test-utils" -sys.path.insert(0, str(test_utils_path)) - -from mock_transport import create_mock_transport, MockTransportCapture -import mock_transport as mt - -# Load environment variables -env_path = Path(__file__).parent.parent.parent.parent / ".env" -load_dotenv(dotenv_path=env_path) - -# Pre-initialize mock transport (required for Python) -mt._mock_transport_capture = MockTransportCapture() - -mock_transport_instance = create_mock_transport() - -# Initialize Sentry with LiteLLM integration -sentry_sdk.init( - traces_sample_rate=1.0, - transport=mock_transport_instance, - send_default_pii=True, - integrations=[LiteLLMIntegration(include_prompts=True)], -) diff --git a/sdks/py/openai-agents/cases/1-simple.py b/sdks/py/openai-agents/cases/1-simple.py deleted file mode 100644 index 691b1ee..0000000 --- a/sdks/py/openai-agents/cases/1-simple.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -1-simple: Basic Completion - -Tests a simple agent completion request with OpenAI Agents SDK -and verifies that Sentry captures the appropriate spans and AI monitoring data. -""" - -import os -from agents import Agent, Runner -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - prompt = inputs["prompt"] - - math_agent = Agent( - name="math_assistant", - instructions=system, - model=model, - ) - - result = await Runner.run(math_agent, prompt) - - if not result.final_output: - raise Exception("No output returned from OpenAI Agents") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {result.final_output}") - - return result.final_output - - -# Export test case functions -test_case = run_test_case("1-simple", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/openai-agents/cases/10-binary-content-redaction.py b/sdks/py/openai-agents/cases/10-binary-content-redaction.py deleted file mode 100644 index 681579f..0000000 --- a/sdks/py/openai-agents/cases/10-binary-content-redaction.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -10-binary-content-redaction: Binary Content Redaction Test - -Tests that when binary data (such as images) is sent to an LLM via OpenAI Agents, -Sentry correctly redacts the binary content in the captured span data and -replaces it with a substitute marker. -""" - -import os -import base64 -from pathlib import Path -from agents import Agent, Runner -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - - # Load static test image - # Path: cases -> openai-agents -> py -> sdks -> repo_root - repo_root = Path(__file__).parent.parent.parent.parent.parent - image_path = repo_root / "shared" / "test-assets" / "test-image-10x10-red.png" - - with open(image_path, "rb") as f: - image_data = f.read() - base64_image = base64.standard_b64encode(image_data).decode("utf-8") - - # Create an agent that can handle images - agent = Agent( - name="image_analyzer", - instructions="You are a helpful assistant that analyzes images.", - model=model, - ) - - # OpenAI Agents SDK accepts either a string or list of ResponseInputItemParam - # For images, we need to wrap content in a message structure and pass as a list - message_with_image = { - "role": "user", - "content": [ - { - "type": "input_image", - "image_url": f"data:image/png;base64,{base64_image}", - "detail": "auto", # Required field: "low", "high", or "auto" - }, - { - "type": "input_text", - "text": "What color is this image? Answer in one word.", - }, - ], - } - - result = await Runner.run(agent, input=[message_with_image]) - - if not result.final_output: - raise Exception("No output returned from OpenAI Agents") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {result.final_output}") - print(f" Sent image with {len(image_data)} bytes of binary data") - - return result.final_output - - -# Export test case functions -test_case = run_test_case("10-binary-content-redaction", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/openai-agents/cases/2-multi-step.py b/sdks/py/openai-agents/cases/2-multi-step.py deleted file mode 100644 index 1a7fb06..0000000 --- a/sdks/py/openai-agents/cases/2-multi-step.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -2-multi-step: Multi-step Conversation - -Tests a multi-step conversation with conversation history using OpenAI Agents SDK -and verifies that Sentry captures all spans for both API calls. -""" - -import os -from agents import Agent, Runner -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - first_prompt = inputs["first_prompt"] - second_prompt = inputs["second_prompt"] - - math_agent = Agent( - name="math_assistant", - instructions=system, - model=model, - ) - - # First call - first_result = await Runner.run(math_agent, first_prompt) - - if not first_result.final_output: - raise Exception("No output returned from OpenAI Agents (first call)") - - first_text = first_result.final_output - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" First response: {first_text}") - - # Second call with conversation history - # Build conversation history from first result's messages - messages = first_result.messages + [ - {"role": "user", "content": second_prompt} - ] - - second_result = await Runner.run( - math_agent, - messages=messages, - ) - - if not second_result.final_output: - raise Exception("No output returned from OpenAI Agents (second call)") - - second_text = second_result.final_output - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Second response: {second_text}") - - return second_text - - -# Export test case functions -test_case = run_test_case("2-multi-step", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/openai-agents/cases/9-message-truncation.py b/sdks/py/openai-agents/cases/9-message-truncation.py deleted file mode 100644 index 25f142a..0000000 --- a/sdks/py/openai-agents/cases/9-message-truncation.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -9-message-truncation: Message Truncation Test - -Tests that when large messages are sent to an LLM via OpenAI Agents, Sentry -correctly tracks the original message count vs. the potentially truncated -message count in the captured span data. -""" - -import os -from agents import Agent, Runner -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - message_size_kb = inputs.get("message_size_kb", 9) - message_count = inputs.get("message_count", 3) - - # Generate large content for the prompt (~9KB each message) - large_content = "x" * (message_size_kb * 1024) - - # Build a large prompt with multiple "messages" embedded - prompt_parts = [] - for i in range(message_count): - prompt_parts.append(f"Message {i + 1}: {large_content}") - - large_prompt = ( - "\n\n".join(prompt_parts) + "\n\nPlease summarize the above messages briefly." - ) - - agent = Agent( - name="summarizer", - instructions="You are a helpful assistant that summarizes content.", - model=model, - ) - - result = await Runner.run(agent, large_prompt) - - if not result.final_output: - raise Exception("No output returned from OpenAI Agents") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {result.final_output[:100]}...") - print( - f" Sent prompt with {message_count} large messages (~{message_size_kb}KB each)" - ) - - return result.final_output - - -# Export test case functions -test_case = run_test_case("9-message-truncation", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/openai-agents/config.json b/sdks/py/openai-agents/config.json deleted file mode 100644 index 17c745f..0000000 --- a/sdks/py/openai-agents/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "sdk_name": "openai-agents", - "framework_type": "agentic", - "overrides": {} -} diff --git a/sdks/py/openai-agents/requirements.txt b/sdks/py/openai-agents/requirements.txt deleted file mode 100644 index 7058933..0000000 --- a/sdks/py/openai-agents/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -sentry-sdk==2.46.0 -openai-agents==0.5.0 -python-dotenv==1.2.1 diff --git a/sdks/py/openai-agents/setup.py b/sdks/py/openai-agents/setup.py deleted file mode 100644 index 85e53f5..0000000 --- a/sdks/py/openai-agents/setup.py +++ /dev/null @@ -1,38 +0,0 @@ -""" -Setup file for OpenAI Agents SDK tests - -Initializes Sentry with OpenAI Agents-specific integrations. -""" - -import os -import sys -from pathlib import Path -from dotenv import load_dotenv -from sentry_sdk.integrations.openai import OpenAIIntegration -from sentry_sdk.integrations.openai_agents import OpenAIAgentsIntegration -import sentry_sdk - -# Add test utils to path -test_utils_path = Path(__file__).parent.parent / "_test-utils" -sys.path.insert(0, str(test_utils_path)) - -from mock_transport import create_mock_transport, MockTransportCapture -import mock_transport as mt - -# Load environment variables -env_path = Path(__file__).parent / ".env" -load_dotenv(dotenv_path=env_path) - -# Pre-initialize mock transport (required for Python) -mt._mock_transport_capture = MockTransportCapture() - -mock_transport_instance = create_mock_transport() - -# Initialize Sentry with OpenAI Agents integration -sentry_sdk.init( - traces_sample_rate=1.0, - transport=mock_transport_instance, - send_default_pii=True, - integrations=[OpenAIAgentsIntegration()], - disabled_integrations=[OpenAIIntegration()], -) diff --git a/sdks/py/openai/cases/1-simple.py b/sdks/py/openai/cases/1-simple.py deleted file mode 100644 index 9da7beb..0000000 --- a/sdks/py/openai/cases/1-simple.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -1-simple: Basic Completion - -Tests a simple chat completion request with OpenAI SDK -and verifies that Sentry captures the appropriate spans and AI monitoring data. -""" - -import os -from openai import OpenAI -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - prompt = inputs["prompt"] - - client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) - - response = client.chat.completions.create( - model=model, - messages=[ - {"role": "system", "content": system}, - {"role": "user", "content": prompt} - ] - ) - - text = response.choices[0].message.content - - if not text: - raise Exception("No output returned from OpenAI") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text}") - - return text - - -# Export test case functions -test_case = run_test_case("1-simple", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/openai/cases/10-binary-content-redaction.py b/sdks/py/openai/cases/10-binary-content-redaction.py deleted file mode 100644 index 777dc01..0000000 --- a/sdks/py/openai/cases/10-binary-content-redaction.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -10-binary-content-redaction: Binary Content Redaction Test - -Tests that when binary data (such as images) is sent to an LLM, Sentry -correctly redacts the binary content in the captured span data and -replaces it with a substitute marker. -""" - -import os -import base64 -from pathlib import Path -from openai import OpenAI -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - - client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) - - # Load static test image - # Path: cases -> openai -> py -> sdks -> repo_root - repo_root = Path(__file__).parent.parent.parent.parent.parent - image_path = repo_root / "shared" / "test-assets" / "test-image-10x10-red.png" - - with open(image_path, "rb") as f: - image_data = f.read() - base64_image = base64.standard_b64encode(image_data).decode("utf-8") - - # Send message with image content (OpenAI vision format) - response = client.chat.completions.create( - model=model, - messages=[ - { - "role": "user", - "content": [ - { - "type": "image_url", - "image_url": { - "url": f"data:image/png;base64,{base64_image}", - }, - }, - { - "type": "text", - "text": "What color is this image? Answer in one word.", - }, - ], - } - ], - ) - - text = response.choices[0].message.content - - if not text: - raise Exception("No output returned from OpenAI") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text}") - print(f" Sent image with {len(image_data)} bytes of binary data") - - return text - - -# Export test case functions -test_case = run_test_case("10-binary-content-redaction", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/openai/cases/2-multi-step.py b/sdks/py/openai/cases/2-multi-step.py deleted file mode 100644 index a591caf..0000000 --- a/sdks/py/openai/cases/2-multi-step.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -2-multi-step: Multi-step Conversation - -Tests a multi-step conversation with conversation history using OpenAI SDK -and verifies that Sentry captures all spans for both API calls. -""" - -import os -from openai import OpenAI -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - first_prompt = inputs["first_prompt"] - second_prompt = inputs["second_prompt"] - - client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) - - # First call - first_response = client.chat.completions.create( - model=model, - messages=[ - {"role": "system", "content": system}, - {"role": "user", "content": first_prompt} - ] - ) - - first_text = first_response.choices[0].message.content - - if not first_text: - raise Exception("No output returned from OpenAI (first call)") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" First response: {first_text}") - - # Second call with conversation history - second_response = client.chat.completions.create( - model=model, - messages=[ - {"role": "system", "content": system}, - {"role": "user", "content": first_prompt}, - {"role": "assistant", "content": first_text}, - {"role": "user", "content": second_prompt} - ] - ) - - second_text = second_response.choices[0].message.content - - if not second_text: - raise Exception("No output returned from OpenAI (second call)") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Second response: {second_text}") - - return second_text - - -# Export test case functions -test_case = run_test_case("2-multi-step", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/openai/cases/9-message-truncation.py b/sdks/py/openai/cases/9-message-truncation.py deleted file mode 100644 index 1c546dc..0000000 --- a/sdks/py/openai/cases/9-message-truncation.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -9-message-truncation: Message Truncation Test - -Tests that when large messages are sent to an LLM, Sentry correctly tracks -the original message count vs. the potentially truncated message count in -the captured span data. -""" - -import os -from openai import OpenAI -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - message_size_kb = inputs.get("message_size_kb", 9) - message_count = inputs.get("message_count", 3) - - client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) - - # Generate large content for each message (~9KB each) - large_content = "x" * (message_size_kb * 1024) - - # Create the messages array with large content - messages = [] - for i in range(message_count): - role = "user" if i % 2 == 0 else "assistant" - messages.append({"role": role, "content": f"Message {i + 1}: {large_content}"}) - - # Ensure the last message is from user (required by OpenAI) - if messages[-1]["role"] == "assistant": - messages.append( - {"role": "user", "content": "Please summarize what we discussed."} - ) - - response = client.chat.completions.create( - model=model, - messages=messages, - ) - - text = response.choices[0].message.content - - if not text: - raise Exception("No output returned from OpenAI") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text[:100]}...") - print( - f" Sent {len(messages)} messages with ~{message_size_kb}KB content each" - ) - - return text - - -# Export test case functions -test_case = run_test_case("9-message-truncation", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/openai/config.json b/sdks/py/openai/config.json deleted file mode 100644 index 28bbbb9..0000000 --- a/sdks/py/openai/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "sdk_name": "openai", - "framework_type": "low-level", - "overrides": {} -} diff --git a/sdks/py/openai/requirements.txt b/sdks/py/openai/requirements.txt deleted file mode 100644 index 924752a..0000000 --- a/sdks/py/openai/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -sentry-sdk==2.46.0 -openai==2.7.2 -python-dotenv==1.2.1 diff --git a/sdks/py/openai/setup.py b/sdks/py/openai/setup.py deleted file mode 100644 index 63dc072..0000000 --- a/sdks/py/openai/setup.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -Setup file for OpenAI SDK tests - -Initializes Sentry with OpenAI-specific integrations. -""" - -import os -import sys -from pathlib import Path -from dotenv import load_dotenv -import sentry_sdk - -# Add test utils to path -test_utils_path = Path(__file__).parent.parent / "_test-utils" -sys.path.insert(0, str(test_utils_path)) - -from mock_transport import create_mock_transport, MockTransportCapture -import mock_transport as mt - -# Load environment variables -env_path = Path(__file__).parent.parent.parent.parent / ".env" -load_dotenv(dotenv_path=env_path) - -# Pre-initialize mock transport (required for Python) -mt._mock_transport_capture = MockTransportCapture() - -mock_transport_instance = create_mock_transport() - -# Initialize Sentry with OpenAI integration (auto-enabled) -sentry_sdk.init( - traces_sample_rate=1.0, - transport=mock_transport_instance, - send_default_pii=True, - # OpenAIIntegration is enabled automatically -) diff --git a/sdks/py/pydantic-ai/cases/1-simple.py b/sdks/py/pydantic-ai/cases/1-simple.py deleted file mode 100644 index 7ae8b22..0000000 --- a/sdks/py/pydantic-ai/cases/1-simple.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -1-simple: Basic Completion - -Tests a simple chat completion request with Pydantic AI SDK -and verifies that Sentry captures the appropriate spans and AI monitoring data. -""" - -import os -from pydantic_ai import Agent -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - prompt = inputs["prompt"] - - # Create agent with system prompt - agent = Agent(f"openai:{model}", system_prompt=system) - - # Run synchronously (inside async function, but using sync method) - result = await agent.run(prompt) - - text = result.output - - if not text: - raise Exception("No output returned from Pydantic AI") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text}") - - return text - - -# Export test case functions -test_case = run_test_case("1-simple", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/pydantic-ai/cases/10-binary-content-redaction.py b/sdks/py/pydantic-ai/cases/10-binary-content-redaction.py deleted file mode 100644 index 4d0a861..0000000 --- a/sdks/py/pydantic-ai/cases/10-binary-content-redaction.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -10-binary-content-redaction: Binary Content Redaction Test - -Tests that when binary data (such as images) is sent to an LLM via Pydantic AI, -Sentry correctly redacts the binary content in the captured span data and -replaces it with a substitute marker. -""" - -import os -from pathlib import Path -from pydantic_ai import Agent, BinaryContent -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - - # Load static test image - # Path: cases -> pydantic-ai -> py -> sdks -> repo_root - repo_root = Path(__file__).parent.parent.parent.parent.parent - image_path = repo_root / "shared" / "test-assets" / "test-image-10x10-red.png" - - with open(image_path, "rb") as f: - image_data = f.read() - - # Create agent for image analysis - agent = Agent( - f"openai:{model}", - system_prompt="You are a helpful assistant that analyzes images.", - ) - - # Pydantic AI uses BinaryContent for images - image_content = BinaryContent(data=image_data, media_type="image/png") - - result = await agent.run( - "What color is this image? Answer in one word.", - message_history=[ - {"role": "user", "content": [image_content]}, - ], - ) - - text = result.output - - if not text: - raise Exception("No output returned from Pydantic AI") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text}") - print(f" Sent image with {len(image_data)} bytes of binary data") - - return text - - -# Export test case functions -test_case = run_test_case("10-binary-content-redaction", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/pydantic-ai/cases/2-multi-step.py b/sdks/py/pydantic-ai/cases/2-multi-step.py deleted file mode 100644 index 882a631..0000000 --- a/sdks/py/pydantic-ai/cases/2-multi-step.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -2-multi-step: Multi-step Conversation - -Tests a multi-step conversation with conversation history using Pydantic AI SDK -and verifies that Sentry captures all spans for both API calls. -""" - -import os -from pydantic_ai import Agent -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - system = inputs["system"] - first_prompt = inputs["first_prompt"] - second_prompt = inputs["second_prompt"] - - # Create agent with system prompt - agent = Agent(f"openai:{model}", system_prompt=system) - - # First call - first_result = await agent.run(first_prompt) - first_text = first_result.output - - if not first_text: - raise Exception("No output returned from Pydantic AI (first call)") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" First response: {first_text}") - - # Second call with conversation history - # Pydantic AI uses message history from the result - second_result = await agent.run( - second_prompt, - message_history=first_result.all_messages() - ) - - second_text = second_result.output - - if not second_text: - raise Exception("No output returned from Pydantic AI (second call)") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Second response: {second_text}") - - return second_text - - -# Export test case functions -test_case = run_test_case("2-multi-step", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/pydantic-ai/cases/9-message-truncation.py b/sdks/py/pydantic-ai/cases/9-message-truncation.py deleted file mode 100644 index 00fe64e..0000000 --- a/sdks/py/pydantic-ai/cases/9-message-truncation.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -9-message-truncation: Message Truncation Test - -Tests that when large messages are sent to an LLM via Pydantic AI, Sentry -correctly tracks the original message count vs. the potentially truncated -message count in the captured span data. -""" - -import os -from pydantic_ai import Agent -from test_runner import run_test_case - - -async def test_logic(inputs): - """The actual test logic""" - model = inputs["model"] - message_size_kb = inputs.get("message_size_kb", 9) - message_count = inputs.get("message_count", 3) - - # Generate large content for each message (~9KB each) - large_content = "x" * (message_size_kb * 1024) - - # Build a large prompt with multiple "messages" embedded - prompt_parts = [] - for i in range(message_count): - prompt_parts.append(f"Message {i + 1}: {large_content}") - - large_prompt = ( - "\n\n".join(prompt_parts) + "\n\nPlease summarize the above messages briefly." - ) - - # Create agent - agent = Agent( - f"openai:{model}", - system_prompt="You are a helpful assistant that summarizes content.", - ) - - result = await agent.run(large_prompt) - - text = result.output - - if not text: - raise Exception("No output returned from Pydantic AI") - - if os.getenv("SENTRY_AI_TEST_VERBOSE") == "true": - print(f" Response: {text[:100]}...") - print( - f" Sent prompt with {message_count} large messages (~{message_size_kb}KB each)" - ) - - return text - - -# Export test case functions -test_case = run_test_case("9-message-truncation", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] diff --git a/sdks/py/pydantic-ai/config.json b/sdks/py/pydantic-ai/config.json deleted file mode 100644 index 54c7b77..0000000 --- a/sdks/py/pydantic-ai/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "sdk_name": "pydantic-ai", - "framework_type": "agentic", - "overrides": {} -} diff --git a/sdks/py/pydantic-ai/requirements.txt b/sdks/py/pydantic-ai/requirements.txt deleted file mode 100644 index ce00765..0000000 --- a/sdks/py/pydantic-ai/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -sentry-sdk==2.46.0 -pydantic-ai==1.12.0 -python-dotenv==1.2.1 diff --git a/sdks/py/pydantic-ai/setup.py b/sdks/py/pydantic-ai/setup.py deleted file mode 100644 index 307f4ea..0000000 --- a/sdks/py/pydantic-ai/setup.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Setup file for Pydantic AI SDK tests - -Initializes Sentry with Pydantic AI-specific integrations. -""" - -import os -import sys -from pathlib import Path -from dotenv import load_dotenv -from sentry_sdk.integrations.pydantic_ai import PydanticAIIntegration -from sentry_sdk.integrations.openai import OpenAIIntegration -import sentry_sdk - -# Add test utils to path -test_utils_path = Path(__file__).parent.parent / "_test-utils" -sys.path.insert(0, str(test_utils_path)) - -from mock_transport import create_mock_transport, MockTransportCapture -import mock_transport as mt - -# Load environment variables -env_path = Path(__file__).parent.parent.parent.parent / ".env" -load_dotenv(dotenv_path=env_path) - -# Pre-initialize mock transport (required for Python) -mt._mock_transport_capture = MockTransportCapture() - -mock_transport_instance = create_mock_transport() - -# Initialize Sentry with Pydantic AI integration -# Disable OpenAI integration to avoid double counting -sentry_sdk.init( - traces_sample_rate=1.0, - transport=mock_transport_instance, - send_default_pii=True, - integrations=[PydanticAIIntegration(include_prompts=True)], - disabled_integrations=[OpenAIIntegration()], -) diff --git a/shared/orchestration/README.md b/shared/orchestration/README.md deleted file mode 100644 index 0010e59..0000000 --- a/shared/orchestration/README.md +++ /dev/null @@ -1,478 +0,0 @@ -# Test Orchestration & CLI - -This directory contains the CLI tool that discovers and runs all SDK test cases. - -## Overview - -The orchestration system provides a unified CLI for running tests across all SDKs (JavaScript and Python). It handles: - -- **Test discovery** - Automatically finds all SDKs and test cases -- **Lifecycle hooks** - Runs setup/teardown functions before/after tests -- **Cross-language support** - Executes both JavaScript and Python tests -- **Filtering** - Run specific SDKs, test cases, or everything -- **Result reporting** - Clear pass/fail output with timing - -## CLI Commands - -### Installation - -```bash -cd shared/orchestration -npm install -npm run build -``` - -### Available Commands - -#### `list` - Show all available tests - -```bash -npm run cli list -``` - -**Output:** -``` -📋 Available SDKs and Test Cases - -js/vercel ✓ - • 1-simple - -py/openai-agents ✓ - • 1-simple - -Total: 2 SDKs, 2 test cases -✓ = has setup.js/setup.py file -``` - -#### `setup` - Install all dependencies - -Install dependencies for the orchestration and all SDKs in the repository. - -```bash -npm run cli setup -``` - -**What it does:** -- Installs orchestration dependencies (`npm install` in `shared/orchestration`) -- For each JavaScript SDK: runs `npm install` -- For each Python SDK: creates `.venv` (if needed) and runs `pip install -r requirements.txt` - -**Output:** -``` -🔧 Setting up Sentry AI SDK Test Repository - -Orchestration - ✓ Installing dependencies... done - -JavaScript SDKs - ✓ js/openai - Installing dependencies... done - ✓ js/vercel - Installing dependencies... done - -Python SDKs - ✓ py/openai-agents - Creating venv... done - ✓ py/openai-agents - Installing dependencies... done - -✓ Setup complete! 7 steps successful -``` - -#### `upgrade` - Upgrade a package across all SDKs - -Upgrade a specific package to a new version across all SDKs that use it. - -```bash -npm run cli upgrade -``` - -**Examples:** - -Upgrade Sentry JavaScript SDK: -```bash -npm run cli upgrade @sentry/node 10.25.0 -``` - -Upgrade Sentry Python SDK: -```bash -npm run cli upgrade sentry-sdk 2.15.0 -``` - -Upgrade OpenAI package: -```bash -npm run cli upgrade openai 6.9.0 -``` - -**What it does:** -1. Detects if package is JavaScript (starts with `@`) or Python -2. Finds all SDKs using that package -3. Updates `package.json` or `requirements.txt` with exact version -4. Runs `npm install` or `pip install` to apply changes - -**Output:** -``` -📦 Upgrading @sentry/node to 10.25.0 - -Detected as JavaScript package - - ✓ js/openai - Updating package.json... done - ✓ js/openai - Installing dependencies... done - ✓ js/vercel - Updating package.json... done - ✓ js/vercel - Installing dependencies... done - -✓ Upgraded 2 SDK(s) successfully -``` - -#### `run` - Execute tests - -**Run all tests:** -```bash -npm run cli run -- --all -``` - -**Run all JavaScript SDKs:** -```bash -npm run cli run -- --sdk js -``` - -**Run all Python SDKs:** -```bash -npm run cli run -- --sdk py -``` - -**Run specific SDK:** -```bash -npm run cli run -- --sdk js/vercel -npm run cli run -- --sdk py/openai-agents -``` - -**Run specific test case across all SDKs:** -```bash -npm run cli run -- --case 1-simple -``` - -**Run specific test case for all JavaScript SDKs:** -```bash -npm run cli run -- --sdk js --case 1-simple -``` - -**Run specific test case for specific SDK:** -```bash -npm run cli run -- --sdk js/vercel --case 1-simple -``` - -**Stop on first failure (fail-fast mode):** -```bash -npm run cli run -- --sdk js/openai --fail-fast -``` - -When an SDK has multiple test cases (e.g., 1-simple, 2-multi-step), fail-fast mode will stop running tests for that SDK after the first failure. - -**Available options:** -- `--all` - Run all tests -- `--sdk ` - Run specific SDK or all SDKs in a language - - `js` - All JavaScript SDKs - - `py` - All Python SDKs - - `js/openai` - Specific SDK -- `--case ` - Run specific test case (e.g., 1-simple) -- `-f, --fail-fast` - Stop SDK tests on first failure -- `-v, --verbose` - Show detailed output including LLM responses -- `-o, --output-dir ` - Output directory for reports (default: ./test-results) -- `-r, --reports ` - Generate reports: ctrf, html, or all (default: all) - -**Note:** `--sdk` can target a language (js/py) or a specific SDK (js/openai). - -**Output example:** -``` -🧪 Running Sentry AI SDK Tests - -Running 1 test case(s) across 1 SDK(s) - -js/vercel (1-simple) - -🔧 Setting up Vercel AI tests... - ✓ Sentry initialized with mock transport - ↻ Resetting test state... - Running 1-simple: Basic Completion - ✓ 1-simple completed - ✓ Cleaning up... -🧹 Tearing down Vercel AI SDK tests... - -📊 Test Results - -js/vercel - ✓ 1-simple (1250ms) - -Summary: - 1 passed, 0 failed, 1 total - Time: 1.25s - -✓ All tests passed! -``` - -## Test Discovery - -The CLI automatically discovers tests by scanning for files in the SDK directory structure: - -### Discovery Pattern - -``` -sdks/ -├── js/ -│ └── {sdk-name}/ -│ ├── setup.js ← Lifecycle hooks (discovered) -│ └── cases/ -│ ├── 1-simple.js ← Test case (discovered) -│ └── 2-*.js ← More test cases (discovered) -└── py/ - └── {sdk-name}/ - ├── setup.py ← Lifecycle hooks (discovered) - └── cases/ - ├── 1-simple.py ← Test case (discovered) - └── 2-*.py ← More test cases (discovered) -``` - -### What Gets Discovered - -**SDKs:** -- Any directory under `sdks/js/` or `sdks/py/` that contains a `cases/` subdirectory -- SDK path format: `{language}/{sdk-name}` (e.g., `js/vercel`, `py/openai-agents`) - -**Test Cases:** -- Any `.js`, `.ts`, or `.py` file inside an SDK's `cases/` directory -- Test case ID is the filename without extension (e.g., `1-simple.js` → `1-simple`) -- Test cases are sorted alphabetically - -**Setup Files:** -- `setup.js`, `setup.ts`, or `setup.py` in the SDK's root directory -- Contains lifecycle hooks: `beforeAll`, `beforeEach`, `afterEach`, `afterAll` - -## Test Execution Flow - -### Lifecycle Sequence - -For each SDK with test cases: - -``` -1. beforeAll() ← Initialize Sentry, load config (once per SDK) -2. For each test case: - a. beforeEach() ← Reset mock transport (before each test) - b. RUN TEST ← Execute test case function - c. afterEach() ← Clean up (after each test) -3. afterAll() ← Teardown Sentry (once per SDK) -``` - -### JavaScript Test Execution - -JavaScript/TypeScript tests are imported and executed directly: - -```javascript -// Test case file: sdks/js/vercel/cases/1-simple.js -module.exports = async function () { - // Test implementation - console.log("Running test..."); - await Sentry.startSpan({ name: "test", op: "test" }, async () => { - // ... test logic - }); - await Sentry.flush(2000); - // ... assertions -}; -``` - -**Execution:** -1. CLI imports the test file as a module -2. Calls the exported function -3. Waits for completion (supports async/await) -4. Catches any thrown errors - -### Python Test Execution - -Python tests require a wrapper script because the orchestration CLI is TypeScript: - -**Architecture:** -``` -TypeScript CLI (orchestration) - ↓ spawns subprocess -Python Test Runner (python-test-runner.py) - ↓ imports and runs -Test Case (1-simple.py) -``` - -**Why the wrapper is needed:** -- TypeScript CLI can't import Python modules directly -- Setup hooks must run in the same Python process as the test -- Sentry SDK state (mock transport) must be shared - -**Python test structure:** -```python -# Test case file: sdks/py/openai-agents/cases/1-simple.py - -async def main(): - """Entry point - runs test logic only""" - print("Running test...") - # ... test implementation - -async def assert_sentry(): - """Validation - checks Sentry captured data""" - # ... assertions -``` - -**Python test runner workflow:** -1. CLI spawns `python-test-runner.py` subprocess -2. Runner imports SDK's `setup.py` module -3. Runner calls `setup.before_all()` -4. Runner calls `setup.before_each()` -5. Runner imports test module and calls `main()` -6. Runner calls `sentry_sdk.flush()` to capture events -7. Runner calls test module's `assert_sentry()` for validation -8. Runner calls `setup.after_each()` -9. Runner calls `setup.after_all()` - -**Important:** Python test cases must have both `main()` and `assert_sentry()` functions. The orchestrator handles the timing of when each is called. - -## Virtual Environment Detection - -For Python SDKs, the orchestrator automatically detects and uses the correct Python interpreter: - -```python -# Check for SDK's .venv -sdkDir/.venv/bin/python ← Uses this if exists -python3 ← Falls back to system Python -``` - -**Best practice:** Always create a `.venv` in each Python SDK directory to ensure correct dependencies. - -## Error Handling - -### Test Failures - -When a test fails: -- Error message and stack trace are captured -- Remaining tests in the SDK continue running -- Final exit code is non-zero (for CI/CD) - -**Example output:** -``` -📊 Test Results - -js/vercel - ✗ 1-simple (850ms) - Fixture validation failed: - - Expected span with op "gen_ai.invoke_agent" not found - -Summary: - 0 passed, 1 failed, 1 total - Time: 0.85s - -✗ Some tests failed -``` - -### Lifecycle Hook Failures - -If `beforeAll` or `afterAll` fails: -- Error is logged -- All tests for that SDK are marked as failed - -If `beforeEach` or `afterEach` fails: -- Specific test case is marked as failed -- `afterEach` runs even if test or `beforeEach` failed - -## Project Structure - -``` -shared/orchestration/ -├── src/ -│ ├── cli.ts # Main CLI entry point, commander setup -│ ├── discovery.ts # Test discovery logic -│ ├── runner.ts # Test execution engine -│ └── types.ts # TypeScript type definitions -├── python-test-runner.py # Python test wrapper script -├── package.json # CLI dependencies -├── tsconfig.json # TypeScript configuration -└── README.md # This file -``` - -## Adding New Commands - -To add a new CLI command, edit `src/cli.ts`: - -```typescript -program - .command('your-command') - .description('What it does') - .option('-o, --option ', 'Option description') - .action(async (options) => { - // Implementation - }); -``` - -## Debugging Test Execution - -### Enable Verbose Output - -Tests log their own output via `console.log`. Both stdout and stderr are inherited from the orchestrator. - -**JavaScript tests:** -```javascript -console.log("Debug: checking spans..."); -``` - -**Python tests:** -```python -print("Debug: checking spans...") -``` - -### Common Issues - -**Issue: "No SDKs found"** -- Check directory structure matches `sdks/{js|py}/{sdk-name}/cases/` -- Ensure test files have `.js`, `.ts`, or `.py` extensions - -**Issue: "Mock transport not initialized" (Python)** -- Ensure `sys.path.insert(0, str(shared_path))` is in `setup.py` -- Check that `create_mock_transport()` is called before `sentry_sdk.init()` - -**Issue: "Test case does not export a default function" (JavaScript)** -- Ensure test file uses `module.exports = async function () { ... }` -- Don't use named exports or ES6 `export default` - -**Issue: Python test can't find dependencies** -- Create `.venv` in SDK directory: `python3 -m venv .venv` -- Install requirements: `source .venv/bin/activate && pip install -r requirements.txt` - -**Issue: Test passes but no Sentry data captured** -- JavaScript: Check `await Sentry.flush(2000)` is called before assertions -- Python: Runner handles flushing automatically between `main()` and `assert_sentry()` - -## Architecture Notes - -### Why TypeScript for Orchestration? - -The orchestration CLI uses TypeScript (not Python) for several reasons: - -1. **Cross-language** - Can spawn both Node.js and Python subprocesses -2. **Modern CLI tools** - Commander.js, Chalk for great UX -3. **Type safety** - Catches errors early -4. **Separation of concerns** - Orchestrator is separate from test code - -### Why ES Modules for Orchestration? - -The orchestration code uses ES modules (`import`/`export`) while SDK tests use CommonJS: - -- **Orchestration** (`shared/orchestration/`): ES modules for modern Node.js features -- **SDK tests** (`sdks/js/*/`): CommonJS for simplicity and compatibility - -This is intentional and prevents module system confusion. - -### Test Isolation - -Each test case should be independent: -- `beforeEach` resets state (clears mock transport) -- Tests should not depend on execution order -- Tests should not share mutable state - -## See Also - -- [Adding SDKs](../../sdks/README.md) - How to implement test cases -- [Test Specifications](../specs/README.md) - Fixture format -- [Test Utilities (JS)](../../sdks/js/_test-utils/README.md) - Mock transport & validation -- [Test Utilities (Python)](../../sdks/py/_test-utils/README.md) - Mock transport & validation -- [Troubleshooting](../../docs/TROUBLESHOOTING.md) - Common pitfalls -- [Main Documentation](../../CLAUDE.md) - Project overview diff --git a/shared/orchestration/js-test-runner.cjs b/shared/orchestration/js-test-runner.cjs deleted file mode 100644 index a9aa7c2..0000000 --- a/shared/orchestration/js-test-runner.cjs +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env node -/** - * JavaScript Test Runner Wrapper - * - * This script runs a single JavaScript SDK test in isolation. - * It's invoked as a subprocess by the orchestration CLI to ensure - * complete isolation between SDKs (Sentry SDK state doesn't leak). - * - * Usage: node js-test-runner.js - */ - -const { resolve, dirname } = require('path'); - -async function main() { - const args = process.argv.slice(2); - - if (args.length < 2) { - console.error('Usage: node js-test-runner.js '); - process.exit(1); - } - - const sdkDir = resolve(args[0]); - const testFilePath = resolve(args[1]); - - try { - // Import setup module - const setupPath = resolve(sdkDir, 'setup.js'); - const setup = require(setupPath); - - // Run beforeAll hook - if (setup.beforeAll) { - await setup.beforeAll(); - } - - // Run beforeEach hook - if (setup.beforeEach) { - await setup.beforeEach(); - } - - // Import and run test - const testModule = require(testFilePath); - if (typeof testModule === 'function') { - await testModule(); - } else { - throw new Error('Test case does not export a function'); - } - - // Run afterEach hook - if (setup.afterEach) { - await setup.afterEach(); - } - - // Run afterAll hook - if (setup.afterAll) { - await setup.afterAll(); - } - - process.exit(0); - } catch (error) { - console.error(`✗ Test failed: ${error.message}`); - if (error.stack) { - console.error(error.stack); - } - process.exit(1); - } -} - -main(); diff --git a/shared/orchestration/package-lock.json b/shared/orchestration/package-lock.json deleted file mode 100644 index 8ddbd6b..0000000 --- a/shared/orchestration/package-lock.json +++ /dev/null @@ -1,2220 +0,0 @@ -{ - "name": "@sentry-ai-sdks/orchestration", - "version": "1.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@sentry-ai-sdks/orchestration", - "version": "1.0.0", - "dependencies": { - "@sentry/node": "^10.27.0", - "chalk": "^5.3.0", - "commander": "^12.1.0", - "ctrf": "^0.0.17", - "dotenv": "^17.2.3", - "glob": "^11.0.0", - "htm": "^3.1.1", - "vhtml": "^2.2.0" - }, - "bin": { - "sentry-ai-test": "dist/cli.js" - }, - "devDependencies": { - "@types/node": "^20.11.0", - "@types/vhtml": "^2.2.9", - "tsx": "^4.7.0", - "typescript": "^5.3.3" - } - }, - "node_modules/@apm-js-collab/code-transformer": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/@apm-js-collab/code-transformer/-/code-transformer-0.8.2.tgz", - "integrity": "sha512-YRjJjNq5KFSjDUoqu5pFUWrrsvGOxl6c3bu+uMFc9HNNptZ2rNU/TI2nLw4jnhQNtka972Ee2m3uqbvDQtPeCA==", - "license": "Apache-2.0" - }, - "node_modules/@apm-js-collab/tracing-hooks": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@apm-js-collab/tracing-hooks/-/tracing-hooks-0.3.1.tgz", - "integrity": "sha512-Vu1CbmPURlN5fTboVuKMoJjbO5qcq9fA5YXpskx3dXe/zTBvjODFoerw+69rVBlRLrJpwPqSDqEuJDEKIrTldw==", - "license": "Apache-2.0", - "dependencies": { - "@apm-js-collab/code-transformer": "^0.8.0", - "debug": "^4.4.1", - "module-details-from-path": "^1.0.4" - } - }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", - "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", - "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", - "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", - "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", - "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", - "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", - "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", - "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", - "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", - "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", - "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", - "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", - "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", - "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", - "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", - "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", - "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", - "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", - "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", - "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", - "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", - "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", - "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", - "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", - "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", - "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@isaacs/balanced-match": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", - "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==", - "license": "MIT", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@isaacs/brace-expansion": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz", - "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==", - "license": "MIT", - "dependencies": { - "@isaacs/balanced-match": "^4.0.1" - }, - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "license": "ISC", - "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@opentelemetry/api": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", - "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", - "license": "Apache-2.0", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/api-logs": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.208.0.tgz", - "integrity": "sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api": "^1.3.0" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/@opentelemetry/context-async-hooks": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.2.0.tgz", - "integrity": "sha512-qRkLWiUEZNAmYapZ7KGS5C4OmBLcP/H2foXeOEaowYCR0wi89fHejrfYfbuLVCMLp/dWZXKvQusdbUEZjERfwQ==", - "license": "Apache-2.0", - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/core": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.2.0.tgz", - "integrity": "sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/instrumentation": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.208.0.tgz", - "integrity": "sha512-Eju0L4qWcQS+oXxi6pgh7zvE2byogAkcsVv0OjHF/97iOz1N/aKE6etSGowYkie+YA1uo6DNwdSxaaNnLvcRlA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api-logs": "0.208.0", - "import-in-the-middle": "^2.0.0", - "require-in-the-middle": "^8.0.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-amqplib": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-amqplib/-/instrumentation-amqplib-0.55.0.tgz", - "integrity": "sha512-5ULoU8p+tWcQw5PDYZn8rySptGSLZHNX/7srqo2TioPnAAcvTy6sQFQXsNPrAnyRRtYGMetXVyZUy5OaX1+IfA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-connect": { - "version": "0.52.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-connect/-/instrumentation-connect-0.52.0.tgz", - "integrity": "sha512-GXPxfNB5szMbV3I9b7kNWSmQBoBzw7MT0ui6iU/p+NIzVx3a06Ri2cdQO7tG9EKb4aKSLmfX9Cw5cKxXqX6Ohg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0", - "@types/connect": "3.4.38" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-dataloader": { - "version": "0.26.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-dataloader/-/instrumentation-dataloader-0.26.0.tgz", - "integrity": "sha512-P2BgnFfTOarZ5OKPmYfbXfDFjQ4P9WkQ1Jji7yH5/WwB6Wm/knynAoA1rxbjWcDlYupFkyT0M1j6XLzDzy0aCA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-express": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-express/-/instrumentation-express-0.57.0.tgz", - "integrity": "sha512-HAdx/o58+8tSR5iW+ru4PHnEejyKrAy9fYFhlEI81o10nYxrGahnMAHWiSjhDC7UQSY3I4gjcPgSKQz4rm/asg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-fs": { - "version": "0.28.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.28.0.tgz", - "integrity": "sha512-FFvg8fq53RRXVBRHZViP+EMxMR03tqzEGpuq55lHNbVPyFklSVfQBN50syPhK5UYYwaStx0eyCtHtbRreusc5g==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-generic-pool": { - "version": "0.52.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-generic-pool/-/instrumentation-generic-pool-0.52.0.tgz", - "integrity": "sha512-ISkNcv5CM2IwvsMVL31Tl61/p2Zm2I2NAsYq5SSBgOsOndT0TjnptjufYVScCnD5ZLD1tpl4T3GEYULLYOdIdQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-graphql": { - "version": "0.56.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-graphql/-/instrumentation-graphql-0.56.0.tgz", - "integrity": "sha512-IPvNk8AFoVzTAM0Z399t34VDmGDgwT6rIqCUug8P9oAGerl2/PEIYMPOl/rerPGu+q8gSWdmbFSjgg7PDVRd3Q==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-hapi": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-hapi/-/instrumentation-hapi-0.55.0.tgz", - "integrity": "sha512-prqAkRf9e4eEpy4G3UcR32prKE8NLNlA90TdEU1UsghOTg0jUvs40Jz8LQWFEs5NbLbXHYGzB4CYVkCI8eWEVQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-http": { - "version": "0.208.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.208.0.tgz", - "integrity": "sha512-rhmK46DRWEbQQB77RxmVXGyjs6783crXCnFjYQj+4tDH/Kpv9Rbg3h2kaNyp5Vz2emF1f9HOQQvZoHzwMWOFZQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/instrumentation": "0.208.0", - "@opentelemetry/semantic-conventions": "^1.29.0", - "forwarded-parse": "2.1.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-ioredis": { - "version": "0.56.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.56.0.tgz", - "integrity": "sha512-XSWeqsd3rKSsT3WBz/JKJDcZD4QYElZEa0xVdX8f9dh4h4QgXhKRLorVsVkK3uXFbC2sZKAS2Ds+YolGwD83Dg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/redis-common": "^0.38.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-kafkajs": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-kafkajs/-/instrumentation-kafkajs-0.18.0.tgz", - "integrity": "sha512-KCL/1HnZN5zkUMgPyOxfGjLjbXjpd4odDToy+7c+UsthIzVLFf99LnfIBE8YSSrYE4+uS7OwJMhvhg3tWjqMBg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.30.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-knex": { - "version": "0.53.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-knex/-/instrumentation-knex-0.53.0.tgz", - "integrity": "sha512-xngn5cH2mVXFmiT1XfQ1aHqq1m4xb5wvU6j9lSgLlihJ1bXzsO543cpDwjrZm2nMrlpddBf55w8+bfS4qDh60g==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.33.1" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-koa": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-koa/-/instrumentation-koa-0.57.0.tgz", - "integrity": "sha512-3JS8PU/D5E3q295mwloU2v7c7/m+DyCqdu62BIzWt+3u9utjxC9QS7v6WmUNuoDN3RM+Q+D1Gpj13ERo+m7CGg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.36.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0" - } - }, - "node_modules/@opentelemetry/instrumentation-lru-memoizer": { - "version": "0.53.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.53.0.tgz", - "integrity": "sha512-LDwWz5cPkWWr0HBIuZUjslyvijljTwmwiItpMTHujaULZCxcYE9eU44Qf/pbVC8TulT0IhZi+RoGvHKXvNhysw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mongodb": { - "version": "0.61.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.61.0.tgz", - "integrity": "sha512-OV3i2DSoY5M/pmLk+68xr5RvkHU8DRB3DKMzYJdwDdcxeLs62tLbkmRyqJZsYf3Ht7j11rq35pHOWLuLzXL7pQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mongoose": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.55.0.tgz", - "integrity": "sha512-5afj0HfF6aM6Nlqgu6/PPHFk8QBfIe3+zF9FGpX76jWPS0/dujoEYn82/XcLSaW5LPUDW8sni+YeK0vTBNri+w==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mysql": { - "version": "0.54.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql/-/instrumentation-mysql-0.54.0.tgz", - "integrity": "sha512-bqC1YhnwAeWmRzy1/Xf9cDqxNG2d/JDkaxnqF5N6iJKN1eVWI+vg7NfDkf52/Nggp3tl1jcC++ptC61BD6738A==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@types/mysql": "2.15.27" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-mysql2": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql2/-/instrumentation-mysql2-0.55.0.tgz", - "integrity": "sha512-0cs8whQG55aIi20gnK8B7cco6OK6N+enNhW0p5284MvqJ5EPi+I1YlWsWXgzv/V2HFirEejkvKiI4Iw21OqDWg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.33.0", - "@opentelemetry/sql-common": "^0.41.2" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-pg": { - "version": "0.61.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.61.0.tgz", - "integrity": "sha512-UeV7KeTnRSM7ECHa3YscoklhUtTQPs6V6qYpG283AB7xpnPGCUCUfECFT9jFg6/iZOQTt3FHkB1wGTJCNZEvPw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.34.0", - "@opentelemetry/sql-common": "^0.41.2", - "@types/pg": "8.15.6", - "@types/pg-pool": "2.0.6" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-redis": { - "version": "0.57.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-redis/-/instrumentation-redis-0.57.0.tgz", - "integrity": "sha512-bCxTHQFXzrU3eU1LZnOZQ3s5LURxQPDlU3/upBzlWY77qOI1GZuGofazj3jtzjctMJeBEJhNwIFEgRPBX1kp/Q==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/redis-common": "^0.38.2", - "@opentelemetry/semantic-conventions": "^1.27.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-tedious": { - "version": "0.27.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-tedious/-/instrumentation-tedious-0.27.0.tgz", - "integrity": "sha512-jRtyUJNZppPBjPae4ZjIQ2eqJbcRaRfJkr0lQLHFmOU/no5A6e9s1OHLd5XZyZoBJ/ymngZitanyRRA5cniseA==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": "^0.208.0", - "@types/tedious": "^4.0.14" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.3.0" - } - }, - "node_modules/@opentelemetry/instrumentation-undici": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.19.0.tgz", - "integrity": "sha512-Pst/RhR61A2OoZQZkn6OLpdVpXp6qn3Y92wXa6umfJe9rV640r4bc6SWvw4pPN6DiQqPu2c8gnSSZPDtC6JlpQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/semantic-conventions": "^1.24.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.7.0" - } - }, - "node_modules/@opentelemetry/redis-common": { - "version": "0.38.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/redis-common/-/redis-common-0.38.2.tgz", - "integrity": "sha512-1BCcU93iwSRZvDAgwUxC/DV4T/406SkMfxGqu5ojc3AvNI+I9GhV7v0J1HljsczuuhcnFLYqD5VmwVXfCGHzxA==", - "license": "Apache-2.0", - "engines": { - "node": "^18.19.0 || >=20.6.0" - } - }, - "node_modules/@opentelemetry/resources": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.2.0.tgz", - "integrity": "sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/sdk-trace-base": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.2.0.tgz", - "integrity": "sha512-xWQgL0Bmctsalg6PaXExmzdedSp3gyKV8mQBwK/j9VGdCDu2fmXIb2gAehBKbkXCpJ4HPkgv3QfoJWRT4dHWbw==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.2.0", - "@opentelemetry/resources": "2.2.0", - "@opentelemetry/semantic-conventions": "^1.29.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.10.0" - } - }, - "node_modules/@opentelemetry/semantic-conventions": { - "version": "1.38.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.38.0.tgz", - "integrity": "sha512-kocjix+/sSggfJhwXqClZ3i9Y/MI0fp7b+g7kCRm6psy2dsf8uApTRclwG18h8Avm7C9+fnt+O36PspJ/OzoWg==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, - "node_modules/@opentelemetry/sql-common": { - "version": "0.41.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/sql-common/-/sql-common-0.41.2.tgz", - "integrity": "sha512-4mhWm3Z8z+i508zQJ7r6Xi7y4mmoJpdvH0fZPFRkWrdp5fq7hhZ2HhYokEOLkfqSMgPR4Z9EyB3DBkbKGOqZiQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "^2.0.0" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.1.0" - } - }, - "node_modules/@prisma/instrumentation": { - "version": "6.19.0", - "resolved": "https://registry.npmjs.org/@prisma/instrumentation/-/instrumentation-6.19.0.tgz", - "integrity": "sha512-QcuYy25pkXM8BJ37wVFBO7Zh34nyRV1GOb2n3lPkkbRYfl4hWl3PTcImP41P0KrzVXfa/45p6eVCos27x3exIg==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/instrumentation": ">=0.52.0 <1" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.8" - } - }, - "node_modules/@sentry/core": { - "version": "10.27.0", - "resolved": "https://registry.npmjs.org/@sentry/core/-/core-10.27.0.tgz", - "integrity": "sha512-Zc68kdH7tWTDtDbV1zWIbo3Jv0fHAU2NsF5aD2qamypKgfSIMSbWVxd22qZyDBkaX8gWIPm/0Sgx6aRXRBXrYQ==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/@sentry/node": { - "version": "10.27.0", - "resolved": "https://registry.npmjs.org/@sentry/node/-/node-10.27.0.tgz", - "integrity": "sha512-1cQZ4+QqV9juW64Jku1SMSz+PoZV+J59lotz4oYFvCNYzex8hRAnDKvNiKW1IVg5mEEkz98mg1fvcUtiw7GTiQ==", - "license": "MIT", - "dependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^2.2.0", - "@opentelemetry/core": "^2.2.0", - "@opentelemetry/instrumentation": "^0.208.0", - "@opentelemetry/instrumentation-amqplib": "0.55.0", - "@opentelemetry/instrumentation-connect": "0.52.0", - "@opentelemetry/instrumentation-dataloader": "0.26.0", - "@opentelemetry/instrumentation-express": "0.57.0", - "@opentelemetry/instrumentation-fs": "0.28.0", - "@opentelemetry/instrumentation-generic-pool": "0.52.0", - "@opentelemetry/instrumentation-graphql": "0.56.0", - "@opentelemetry/instrumentation-hapi": "0.55.0", - "@opentelemetry/instrumentation-http": "0.208.0", - "@opentelemetry/instrumentation-ioredis": "0.56.0", - "@opentelemetry/instrumentation-kafkajs": "0.18.0", - "@opentelemetry/instrumentation-knex": "0.53.0", - "@opentelemetry/instrumentation-koa": "0.57.0", - "@opentelemetry/instrumentation-lru-memoizer": "0.53.0", - "@opentelemetry/instrumentation-mongodb": "0.61.0", - "@opentelemetry/instrumentation-mongoose": "0.55.0", - "@opentelemetry/instrumentation-mysql": "0.54.0", - "@opentelemetry/instrumentation-mysql2": "0.55.0", - "@opentelemetry/instrumentation-pg": "0.61.0", - "@opentelemetry/instrumentation-redis": "0.57.0", - "@opentelemetry/instrumentation-tedious": "0.27.0", - "@opentelemetry/instrumentation-undici": "0.19.0", - "@opentelemetry/resources": "^2.2.0", - "@opentelemetry/sdk-trace-base": "^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0", - "@prisma/instrumentation": "6.19.0", - "@sentry/core": "10.27.0", - "@sentry/node-core": "10.27.0", - "@sentry/opentelemetry": "10.27.0", - "import-in-the-middle": "^2", - "minimatch": "^9.0.0" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@sentry/node-core": { - "version": "10.27.0", - "resolved": "https://registry.npmjs.org/@sentry/node-core/-/node-core-10.27.0.tgz", - "integrity": "sha512-Dzo1I64Psb7AkpyKVUlR9KYbl4wcN84W4Wet3xjLmVKMgrCo2uAT70V4xIacmoMH5QLZAx0nGfRy9yRCd4nzBg==", - "license": "MIT", - "dependencies": { - "@apm-js-collab/tracing-hooks": "^0.3.1", - "@sentry/core": "10.27.0", - "@sentry/opentelemetry": "10.27.0", - "import-in-the-middle": "^2" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/core": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/instrumentation": ">=0.57.1 <1", - "@opentelemetry/resources": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0" - } - }, - "node_modules/@sentry/node/node_modules/minimatch": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", - "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@sentry/opentelemetry": { - "version": "10.27.0", - "resolved": "https://registry.npmjs.org/@sentry/opentelemetry/-/opentelemetry-10.27.0.tgz", - "integrity": "sha512-z2vXoicuGiqlRlgL9HaYJgkin89ncMpNQy0Kje6RWyhpzLe8BRgUXlgjux7WrSrcbopDdC1OttSpZsJ/Wjk7fg==", - "license": "MIT", - "dependencies": { - "@sentry/core": "10.27.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/context-async-hooks": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/core": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0 || ^2.2.0", - "@opentelemetry/semantic-conventions": "^1.37.0" - } - }, - "node_modules/@types/connect": { - "version": "3.4.38", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", - "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/mysql": { - "version": "2.15.27", - "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.27.tgz", - "integrity": "sha512-YfWiV16IY0OeBfBCk8+hXKmdTKrKlwKN1MNKAPBu5JYxLwBEZl7QzeEpGnlZb3VMGJrrGmB84gXiH+ofs/TezA==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/node": { - "version": "20.19.24", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.24.tgz", - "integrity": "sha512-FE5u0ezmi6y9OZEzlJfg37mqqf6ZDSF2V/NLjUyGrR9uTZ7Sb9F7bLNZ03S4XVUNRWGA7Ck4c1kK+YnuWjl+DA==", - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, - "node_modules/@types/pg": { - "version": "8.15.6", - "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.15.6.tgz", - "integrity": "sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "pg-protocol": "*", - "pg-types": "^2.2.0" - } - }, - "node_modules/@types/pg-pool": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@types/pg-pool/-/pg-pool-2.0.6.tgz", - "integrity": "sha512-TaAUE5rq2VQYxab5Ts7WZhKNmuN78Q6PiFonTDdpbx8a1H0M1vhy3rhiMjl+e2iHmogyMw7jZF4FrE6eJUy5HQ==", - "license": "MIT", - "dependencies": { - "@types/pg": "*" - } - }, - "node_modules/@types/tedious": { - "version": "4.0.14", - "resolved": "https://registry.npmjs.org/@types/tedious/-/tedious-4.0.14.tgz", - "integrity": "sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/vhtml": { - "version": "2.2.9", - "resolved": "https://registry.npmjs.org/@types/vhtml/-/vhtml-2.2.9.tgz", - "integrity": "sha512-maEIRJb+PdK2FWDORl0a0aNUSGlniMl8pN+7WbanLzx1Gry4gLfsT0C9O/6JucPPBHCNrqDSImr2QcsUENLKUg==", - "dev": true, - "license": "MIT" - }, - "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", - "license": "MIT", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-import-attributes": { - "version": "1.9.5", - "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", - "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", - "license": "MIT", - "peerDependencies": { - "acorn": "^8" - } - }, - "node_modules/ajv": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", - "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ajv-formats": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", - "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", - "license": "MIT", - "dependencies": { - "ajv": "^8.0.0" - }, - "peerDependencies": { - "ajv": "^8.0.0" - }, - "peerDependenciesMeta": { - "ajv": { - "optional": true - } - } - }, - "node_modules/ansi-regex": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", - "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/ansi-styles": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", - "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "license": "MIT" - }, - "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/chalk": { - "version": "5.6.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", - "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", - "license": "MIT", - "engines": { - "node": "^12.17.0 || ^14.13 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/cjs-module-lexer": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", - "integrity": "sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==", - "license": "MIT" - }, - "node_modules/cliui": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", - "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==", - "license": "ISC", - "dependencies": { - "string-width": "^7.2.0", - "strip-ansi": "^7.1.0", - "wrap-ansi": "^9.0.0" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/cliui/node_modules/emoji-regex": { - "version": "10.6.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", - "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", - "license": "MIT" - }, - "node_modules/cliui/node_modules/string-width": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", - "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^10.3.0", - "get-east-asian-width": "^1.0.0", - "strip-ansi": "^7.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/cliui/node_modules/wrap-ansi": { - "version": "9.0.2", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", - "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.2.1", - "string-width": "^7.0.0", - "strip-ansi": "^7.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "license": "MIT" - }, - "node_modules/commander": { - "version": "12.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", - "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/ctrf": { - "version": "0.0.17", - "resolved": "https://registry.npmjs.org/ctrf/-/ctrf-0.0.17.tgz", - "integrity": "sha512-PPk9b+AuA+UoBcbzSQWXMIuh5601zDHgXlmHIG8ESxTUnnb0eM2sz8H3jQLYQZTpyIaZVCLFZFslIDB1EMVZ1g==", - "license": "MIT", - "dependencies": { - "ajv": "^8.17.1", - "ajv-formats": "^3.0.1", - "glob": "^11.0.3", - "typescript": "^5.8.3", - "yargs": "^18.0.0" - }, - "bin": { - "ctrf": "dist/cli.js" - }, - "engines": { - "node": ">=20.19.0" - } - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/dotenv": { - "version": "17.2.3", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", - "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", - "license": "MIT" - }, - "node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", - "license": "MIT" - }, - "node_modules/esbuild": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", - "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.25.12", - "@esbuild/android-arm": "0.25.12", - "@esbuild/android-arm64": "0.25.12", - "@esbuild/android-x64": "0.25.12", - "@esbuild/darwin-arm64": "0.25.12", - "@esbuild/darwin-x64": "0.25.12", - "@esbuild/freebsd-arm64": "0.25.12", - "@esbuild/freebsd-x64": "0.25.12", - "@esbuild/linux-arm": "0.25.12", - "@esbuild/linux-arm64": "0.25.12", - "@esbuild/linux-ia32": "0.25.12", - "@esbuild/linux-loong64": "0.25.12", - "@esbuild/linux-mips64el": "0.25.12", - "@esbuild/linux-ppc64": "0.25.12", - "@esbuild/linux-riscv64": "0.25.12", - "@esbuild/linux-s390x": "0.25.12", - "@esbuild/linux-x64": "0.25.12", - "@esbuild/netbsd-arm64": "0.25.12", - "@esbuild/netbsd-x64": "0.25.12", - "@esbuild/openbsd-arm64": "0.25.12", - "@esbuild/openbsd-x64": "0.25.12", - "@esbuild/openharmony-arm64": "0.25.12", - "@esbuild/sunos-x64": "0.25.12", - "@esbuild/win32-arm64": "0.25.12", - "@esbuild/win32-ia32": "0.25.12", - "@esbuild/win32-x64": "0.25.12" - } - }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "license": "MIT" - }, - "node_modules/fast-uri": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", - "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "BSD-3-Clause" - }, - "node_modules/foreground-child": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", - "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", - "license": "ISC", - "dependencies": { - "cross-spawn": "^7.0.6", - "signal-exit": "^4.0.1" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/forwarded-parse": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/forwarded-parse/-/forwarded-parse-2.1.2.tgz", - "integrity": "sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==", - "license": "MIT" - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "license": "ISC", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "node_modules/get-east-asian-width": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.4.0.tgz", - "integrity": "sha512-QZjmEOC+IT1uk6Rx0sX22V6uHWVwbdbxf1faPqJ1QhLdGgsRGCZoyaQBm/piRdJy/D2um6hM1UP7ZEeQ4EkP+Q==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/get-tsconfig": { - "version": "4.13.0", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz", - "integrity": "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "resolve-pkg-maps": "^1.0.0" - }, - "funding": { - "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" - } - }, - "node_modules/glob": { - "version": "11.0.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.3.tgz", - "integrity": "sha512-2Nim7dha1KVkaiF4q6Dj+ngPPMdfvLJEOpZk/jKiUAkqKebpGAWQXAq9z1xu9HKu5lWfqw/FASuccEjyznjPaA==", - "license": "ISC", - "dependencies": { - "foreground-child": "^3.3.1", - "jackspeak": "^4.1.1", - "minimatch": "^10.0.3", - "minipass": "^7.1.2", - "package-json-from-dist": "^1.0.0", - "path-scurry": "^2.0.0" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/htm": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/htm/-/htm-3.1.1.tgz", - "integrity": "sha512-983Vyg8NwUE7JkZ6NmOqpCZ+sh1bKv2iYTlUkzlWmA5JD2acKoxd4KVxbMmxX/85mtfdnDmTFoNKcg5DGAvxNQ==", - "license": "Apache-2.0" - }, - "node_modules/import-in-the-middle": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-2.0.0.tgz", - "integrity": "sha512-yNZhyQYqXpkT0AKq3F3KLasUSK4fHvebNH5hOsKQw2dhGSALvQ4U0BqUc5suziKvydO5u5hgN2hy1RJaho8U5A==", - "license": "Apache-2.0", - "dependencies": { - "acorn": "^8.14.0", - "acorn-import-attributes": "^1.9.5", - "cjs-module-lexer": "^1.2.2", - "module-details-from-path": "^1.0.3" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "license": "ISC" - }, - "node_modules/jackspeak": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.1.tgz", - "integrity": "sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ==", - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "license": "MIT" - }, - "node_modules/lru-cache": { - "version": "11.2.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.2.tgz", - "integrity": "sha512-F9ODfyqML2coTIsQpSkRHnLSZMtkU8Q+mSfcaIyKwy58u+8k5nvAYeiNhsyMARvzNcXJ9QfWVrcPsC9e9rAxtg==", - "license": "ISC", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/minimatch": { - "version": "10.1.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", - "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", - "license": "BlueOak-1.0.0", - "dependencies": { - "@isaacs/brace-expansion": "^5.0.0" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "license": "ISC", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/module-details-from-path": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", - "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", - "license": "MIT" - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/package-json-from-dist": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", - "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", - "license": "BlueOak-1.0.0" - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-scurry": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.0.tgz", - "integrity": "sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==", - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^11.0.0", - "minipass": "^7.1.2" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/pg-int8": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", - "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", - "license": "ISC", - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/pg-protocol": { - "version": "1.10.3", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.10.3.tgz", - "integrity": "sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==", - "license": "MIT" - }, - "node_modules/pg-types": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", - "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", - "license": "MIT", - "dependencies": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-array": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", - "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-bytea": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", - "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-date": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", - "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-interval": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", - "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", - "license": "MIT", - "dependencies": { - "xtend": "^4.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-in-the-middle": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-8.0.1.tgz", - "integrity": "sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==", - "license": "MIT", - "dependencies": { - "debug": "^4.3.5", - "module-details-from-path": "^1.0.3" - }, - "engines": { - "node": ">=9.3.0 || >=8.10.0 <9.0.0" - } - }, - "node_modules/resolve-pkg-maps": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", - "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" - } - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "license": "MIT", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/string-width-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/string-width-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz", - "integrity": "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^6.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/tsx": { - "version": "4.20.6", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.6.tgz", - "integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==", - "dev": true, - "license": "MIT", - "dependencies": { - "esbuild": "~0.25.0", - "get-tsconfig": "^4.7.5" - }, - "bin": { - "tsx": "dist/cli.mjs" - }, - "engines": { - "node": ">=18.0.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - } - }, - "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "license": "MIT" - }, - "node_modules/vhtml": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/vhtml/-/vhtml-2.2.0.tgz", - "integrity": "sha512-TPXrXrxBOslRUVnlVkiAqhoXneiertIg86bdvzionrUYhEuiROvyPZNiiP6GIIJ2Q7oPNVyEtIx8gMAZZE9lCQ==", - "license": "MIT" - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs": { - "name": "wrap-ansi", - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/wrap-ansi-cjs/node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "license": "MIT", - "engines": { - "node": ">=0.4" - } - }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs": { - "version": "18.0.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.0.0.tgz", - "integrity": "sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg==", - "license": "MIT", - "dependencies": { - "cliui": "^9.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "string-width": "^7.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^22.0.0" - }, - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=23" - } - }, - "node_modules/yargs-parser": { - "version": "22.0.0", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", - "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", - "license": "ISC", - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=23" - } - }, - "node_modules/yargs/node_modules/emoji-regex": { - "version": "10.6.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", - "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", - "license": "MIT" - }, - "node_modules/yargs/node_modules/string-width": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", - "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^10.3.0", - "get-east-asian-width": "^1.0.0", - "strip-ansi": "^7.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - } - } -} diff --git a/shared/orchestration/package.json b/shared/orchestration/package.json deleted file mode 100644 index d201502..0000000 --- a/shared/orchestration/package.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "name": "@sentry-ai-sdks/orchestration", - "version": "1.0.0", - "description": "CLI tool for running Sentry AI SDK integration tests", - "type": "module", - "bin": { - "sentry-ai-test": "./dist/cli.js" - }, - "scripts": { - "build": "tsc", - "dev": "tsx src/cli.ts", - "cli": "tsx src/cli.ts" - }, - "keywords": [ - "sentry", - "ai", - "testing", - "cli" - ], - "dependencies": { - "@sentry/node": "^10.27.0", - "chalk": "^5.3.0", - "commander": "^12.1.0", - "ctrf": "^0.0.17", - "dotenv": "^17.2.3", - "glob": "^11.0.0", - "htm": "^3.1.1", - "vhtml": "^2.2.0" - }, - "devDependencies": { - "@types/node": "^20.11.0", - "@types/vhtml": "^2.2.9", - "tsx": "^4.7.0", - "typescript": "^5.3.3" - } -} diff --git a/shared/orchestration/python-test-runner.py b/shared/orchestration/python-test-runner.py deleted file mode 100644 index 4dcec28..0000000 --- a/shared/orchestration/python-test-runner.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 -""" -Python Test Runner - Executes Python test cases with lifecycle hooks - -This script is used by the TypeScript orchestration tool to run Python tests. -It properly handles setup.py lifecycle hooks in the same Python process as the test. - -Usage: - python python-test-runner.py -""" - -import sys -import asyncio -from pathlib import Path - - -def main(): - if len(sys.argv) < 3: - print("Usage: python python-test-runner.py ") - sys.exit(1) - - sdk_path = Path(sys.argv[1]).resolve() - test_file_path = Path(sys.argv[2]).resolve() - - # Add SDK directory to path so we can import setup - sys.path.insert(0, str(sdk_path)) - - # Import setup module - import setup - - try: - # Run beforeAll hook - if hasattr(setup, "before_all"): - setup.before_all() - - # Run beforeEach hook - if hasattr(setup, "before_each"): - setup.before_each() - - # Import and run the test case - # We need to add the cases directory to path temporarily - cases_dir = test_file_path.parent - sys.path.insert(0, str(cases_dir)) - - # Import the test module - test_module_name = test_file_path.stem - test_module = __import__(test_module_name) - - # Run the test function - let it handle its own logic - if not hasattr(test_module, "main"): - print(f"ERROR: Test module {test_module_name} has no main() function", file=sys.stderr) - sys.exit(1) - - # Create a transaction for this test (like JS tests do) - import sentry_sdk - with sentry_sdk.start_transaction(op="test", name=f"{test_module_name}"): - if asyncio.iscoroutinefunction(test_module.main): - asyncio.run(test_module.main()) - else: - test_module.main() - - # Flush Sentry to ensure all events are sent to transport - sentry_sdk.flush(timeout=2.0) - - # Run assertions if they exist (should be called after transaction completes) - if hasattr(test_module, "assert_sentry"): - if asyncio.iscoroutinefunction(test_module.assert_sentry): - asyncio.run(test_module.assert_sentry()) - else: - test_module.assert_sentry() - - # Run afterEach hook - if hasattr(setup, "after_each"): - setup.after_each() - - # Run afterAll hook - if hasattr(setup, "after_all"): - setup.after_all() - - except Exception as e: - # Ensure cleanup hooks run even on failure - if hasattr(setup, "after_each"): - try: - setup.after_each() - except Exception: - pass - - if hasattr(setup, "after_all"): - try: - setup.after_all() - except Exception: - pass - - # Print error message cleanly without full traceback - print(f"\n✗ Test failed: {str(e)}", file=sys.stderr) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/shared/orchestration/src/cli.ts b/shared/orchestration/src/cli.ts deleted file mode 100644 index 5d0fc89..0000000 --- a/shared/orchestration/src/cli.ts +++ /dev/null @@ -1,259 +0,0 @@ -#!/usr/bin/env node - -/** - * Sentry AI SDK Integration Test CLI - * - * Main entry point for the test orchestration system - */ - -import { config } from "dotenv"; -import { dirname, join, resolve } from "path"; -import { fileURLToPath } from "url"; -import { Command } from "commander"; -import chalk from "chalk"; -import { discoverSDKs, filterSDKs } from "./discovery.js"; -import { runTests } from "./runner.js"; -import { setup } from "./setup.js"; -import { upgrade } from "./upgrade.js"; -import { - generateCTRFReport, - writeCTRFFile, - printCTRFReport, - generateHTML, - writeHTMLFile, -} from "./reporters/index.js"; -import type { TestResult } from "./types.js"; - -// Load root .env file -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); -// Navigate from src/ -> orchestration/ -> shared/ -> root/ -const rootDir = join(__dirname, "../../.."); -config({ path: join(rootDir, ".env") }); - -const program = new Command(); - -program - .name("sentry-ai-test") - .description("CLI tool for running Sentry AI SDK integration tests") - .version("1.0.0"); - -/** - * List command - shows all available SDKs and test cases - */ -program - .command("list") - .description("List all available SDKs and test cases") - .action(async () => { - console.log(chalk.blue.bold("\n📋 Available SDKs and Test Cases\n")); - - const sdks = await discoverSDKs(); - - if (sdks.length === 0) { - console.log(chalk.yellow("No SDKs found. Have you implemented any yet?")); - console.log( - chalk.gray("SDKs should be in: sdks/{js|py}/{sdk-name}/cases/") - ); - return; - } - - for (const sdk of sdks) { - const setupIndicator = sdk.hasSetup ? chalk.green("✓") : chalk.gray("○"); - console.log(chalk.cyan.bold(`${sdk.path}`) + ` ${setupIndicator}`); - - if (sdk.cases.length === 0) { - console.log(chalk.gray(" No test cases found")); - } else { - for (const testCase of sdk.cases) { - console.log(chalk.gray(` • ${testCase.id}`)); - } - } - console.log(""); - } - - console.log( - chalk.gray( - `Total: ${sdks.length} SDKs, ${sdks.reduce( - (sum, sdk) => sum + sdk.cases.length, - 0 - )} test cases` - ) - ); - console.log( - chalk.gray(`${chalk.green("✓")} = has setup.ts/setup.py file\n`) - ); - }); - -/** - * Run command - executes test cases - */ -program - .command("run [filter]") - .description("Run test cases") - .option("-c, --case ", "Run specific test case (e.g., 1-simple)") - .option("-a, --all", "Run all tests") - .option("-v, --verbose", "Show detailed output including LLM responses") - .option("-f, --fail-fast", "Stop SDK tests on first failure") - .option( - "-o, --output-dir ", - "Output directory for reports", - "./test-results" - ) - .option( - "-r, --reports ", - 'Generate reports (comma-separated: ctrf,html or "all")', - "all" - ) - .option("--local-sentry-python ", "Use local Sentry Python SDK (sentry-python)") - .option("--local-sentry-javascript ", "Use local Sentry JavaScript SDK (sentry-javascript)") - .action(async (filter, options) => { - // Validate options - if (!filter && !options.case && !options.all) { - console.log(chalk.red("Error: You must specify a filter, --case, or --all")); - console.log(chalk.gray("Examples:")); - console.log( - chalk.gray(" npm run cli run js (all JS SDKs)") - ); - console.log( - chalk.gray(" npm run cli run py (all Python SDKs)") - ); - console.log( - chalk.gray(" npm run cli run langchain (js/langchain + py/langchain)") - ); - console.log( - chalk.gray(" npm run cli run pydantic-ai (py/pydantic-ai only)") - ); - console.log( - chalk.gray(" npm run cli run js/openai (specific SDK)") - ); - console.log(chalk.gray(" npm run cli run -- --case 1-simple")); - console.log(chalk.gray(" npm run cli run -- --all")); - process.exit(1); - } - - console.log(chalk.blue.bold("\n🧪 Running Sentry AI SDK Tests\n")); - - // Discover all SDKs - const allSDKs = await discoverSDKs(); - - if (allSDKs.length === 0) { - console.log(chalk.yellow("No SDKs found. Have you implemented any yet?")); - return; - } - - // Filter based on options - const sdks = filterSDKs(allSDKs, { - filter, - case: options.case, - }); - - if (sdks.length === 0) { - console.log(chalk.yellow("No SDKs or test cases match your filters.")); - return; - } - - // Show what we're running - const totalCases = sdks.reduce((sum, sdk) => sum + sdk.cases.length, 0); - console.log( - chalk.gray( - `Running ${totalCases} test case(s) across ${sdks.length} SDK(s)\n` - ) - ); - - for (const sdk of sdks) { - console.log( - chalk.cyan(`${sdk.path}`) + - chalk.gray(` (${sdk.cases.map((c) => c.id).join(", ")})`) - ); - } - console.log(""); - - // Set verbose flag in environment for test runners - if (options.verbose) { - process.env.SENTRY_AI_TEST_VERBOSE = "true"; - } - - // Run tests - const startTime = Date.now(); - const results = await runTests(sdks, { - localSentryPythonPath: options.localSentryPython, - localSentryJavaScriptPath: options.localSentryJavascript, - failFast: options.failFast - }); - const duration = Date.now() - startTime; - - // Generate CTRF report (single source of truth) - const ctrfReport = generateCTRFReport(results, sdks, duration); - - // Print to console (always) - printCTRFReport(ctrfReport); - - // Generate file reports if requested - const reportFormats = options.reports - .toLowerCase() - .split(",") - .map((f: string) => f.trim()); - const generateAll = reportFormats.includes("all"); - const generateCTRF = generateAll || reportFormats.includes("ctrf"); - const generateHTMLReport = generateAll || reportFormats.includes("html"); - - if (generateCTRF || generateHTMLReport) { - const outputDir = options.outputDir; - - if (generateCTRF) { - const ctrfPath = await writeCTRFFile(ctrfReport, outputDir); - console.log(chalk.gray(`📄 CTRF report: ${resolve(ctrfPath)}`)); - } - - if (generateHTMLReport) { - const html = generateHTML(ctrfReport); - const htmlPath = await writeHTMLFile(html, outputDir); - console.log(chalk.gray(`📄 HTML report: ${resolve(htmlPath)}`)); - } - - console.log(""); - } - - // Exit with error code if any tests failed - const hasFailures = results.some((r) => r.status === "failed"); - process.exit(hasFailures ? 1 : 0); - }); - -/** - * Setup command - Install all dependencies - */ -program - .command("setup [language]") - .description("Install all dependencies across the repository (optionally filter by 'js' or 'py')") - .option("--local-sentry-python ", "Use local Sentry Python SDK (sentry-python)") - .option("--local-sentry-javascript ", "Use local Sentry JavaScript SDK (sentry-javascript)") - .action(async (language, options) => { - // Validate language argument if provided - if (language && language !== 'js' && language !== 'py') { - console.log(chalk.red(`Error: Invalid language "${language}". Must be "js" or "py".`)); - console.log(chalk.gray('Examples:')); - console.log(chalk.gray(' npm run cli setup (all SDKs)')); - console.log(chalk.gray(' npm run cli setup js (JS only)')); - console.log(chalk.gray(' npm run cli setup py (Python only)')); - process.exit(1); - } - - await setup({ - language: language as 'js' | 'py' | undefined, - localSentryPythonPath: options.localSentryPython, - localSentryJavaScriptPath: options.localSentryJavascript - }); - }); - -/** - * Upgrade command - Upgrade a package across all SDKs - */ -program - .command("upgrade ") - .description("Upgrade a package to a specific version across all SDKs") - .action(async (packageName: string, version: string) => { - await upgrade(packageName, version); - }); - -// Parse command line arguments -program.parse(); diff --git a/shared/orchestration/src/discovery.ts b/shared/orchestration/src/discovery.ts deleted file mode 100644 index ec422a2..0000000 --- a/shared/orchestration/src/discovery.ts +++ /dev/null @@ -1,196 +0,0 @@ -/** - * Test discovery logic - finds SDKs and test cases - */ - -import { glob } from 'glob'; -import { basename, dirname, join, relative } from 'path'; -import { fileURLToPath } from 'url'; -import { existsSync, readFileSync } from 'fs'; -import type { SDK, TestCase, SDKConfig } from './types.js'; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); - -// Root directory of the repository -export const REPO_ROOT = join(__dirname, '../../..'); - -/** - * Load SDK config.json if it exists - */ -function loadSDKConfig(absolutePath: string): SDKConfig | undefined { - const configPath = join(absolutePath, 'config.json'); - - if (!existsSync(configPath)) { - return undefined; - } - - try { - const content = readFileSync(configPath, 'utf-8'); - return JSON.parse(content) as SDKConfig; - } catch (error) { - console.warn(`Warning: Failed to load SDK config at ${configPath}:`, error); - return undefined; - } -} - -/** - * Natural sort comparator that handles numeric prefixes correctly. - * E.g., "1-simple" < "2-multi" < "10-binary" (not lexical "1" < "10" < "2") - */ -function naturalSortCompare(a: string, b: string): number { - // Extract numeric prefix if present (e.g., "10-binary" -> 10) - const aMatch = a.match(/^(\d+)/); - const bMatch = b.match(/^(\d+)/); - - // If both have numeric prefixes, compare numerically - if (aMatch && bMatch) { - const aNum = parseInt(aMatch[1], 10); - const bNum = parseInt(bMatch[1], 10); - if (aNum !== bNum) { - return aNum - bNum; - } - // If numeric prefixes are equal, compare the rest lexically - return a.localeCompare(b); - } - - // If only one has a numeric prefix, it comes first - if (aMatch) return -1; - if (bMatch) return 1; - - // Neither has a numeric prefix, compare lexically - return a.localeCompare(b); -} - -/** - * Discovers all SDKs and their test cases - */ -export async function discoverSDKs(): Promise { - const sdks: SDK[] = []; - - // Find all case files in sdks/ - const casePattern = join(REPO_ROOT, 'sdks/*/*/cases/*.{ts,js,py}'); - const caseFiles = await glob(casePattern); - - // Group by SDK - const sdkMap = new Map(); - - for (const caseFile of caseFiles) { - const rel = relative(join(REPO_ROOT, 'sdks'), caseFile); - const parts = rel.split('/'); - - if (parts.length < 4) continue; // Should be: language/sdk-name/cases/case-file - - const language = parts[0] as 'js' | 'py'; - const sdkName = parts[1]; - const sdkPath = `${language}/${sdkName}`; - const absolutePath = join(REPO_ROOT, 'sdks', language, sdkName); - - if (!sdkMap.has(sdkPath)) { - sdkMap.set(sdkPath, { - files: [], - language, - name: sdkName, - absolutePath - }); - } - - sdkMap.get(sdkPath)!.files.push(caseFile); - } - - // Convert to SDK objects - for (const [sdkPath, data] of sdkMap) { - const cases: TestCase[] = data.files.map(filePath => { - const fileName = basename(filePath); - const caseId = fileName.replace(/\.(ts|js|py)$/, ''); - - return { - id: caseId, - filePath, - sdkPath - }; - }); - - // Check if setup file exists - const setupExtensions = data.language === 'js' ? ['.ts', '.js'] : ['.py']; - const hasSetup = setupExtensions.some(ext => - existsSync(join(data.absolutePath, `setup${ext}`)) - ); - - // Load SDK config if it exists - const config = loadSDKConfig(data.absolutePath); - - sdks.push({ - language: data.language, - name: data.name, - path: sdkPath, - absolutePath: data.absolutePath, - cases: cases.sort((a, b) => naturalSortCompare(a.id, b.id)), - hasSetup, - config - }); - } - - return sdks.sort((a, b) => a.path.localeCompare(b.path)); -} - -/** - * Filter SDKs based on options - */ -export function filterSDKs(sdks: SDK[], options: { filter?: string, case?: string }): SDK[] { - let filtered = sdks; - - // Filter by SDK filter (positional argument) - if (options.filter) { - const filterValue = options.filter; - - // Check if it's a language-only filter (e.g., "js" or "py") - if (filterValue === 'js' || filterValue === 'py') { - filtered = filtered.filter(sdk => sdk.language === filterValue); - } - // Check if it's an exact path match (e.g., "js/openai") - else if (filterValue.includes('/')) { - filtered = filtered.filter(sdk => sdk.path === filterValue); - } - // Otherwise, filter by SDK name (partial match across both languages) - // e.g., "lang" matches "langchain" and "langgraph" in both js and py - // e.g., "langchain" matches "js/langchain" and "py/langchain" - // e.g., "pydantic-ai" matches only "py/pydantic-ai" - else { - filtered = filtered.filter(sdk => sdk.name.includes(filterValue)); - } - } - - // Filter by case - if (options.case) { - filtered = filtered.map(sdk => ({ - ...sdk, - cases: sdk.cases.filter(c => c.id === options.case) - })).filter(sdk => sdk.cases.length > 0); - } - - return filtered; -} - -/** - * Load lifecycle hooks from setup file - */ -export async function loadSetupHooks(sdkPath: string): Promise { - const setupExtensions = sdkPath.startsWith('js/') ? ['.ts', '.js'] : ['.py']; - - for (const ext of setupExtensions) { - const setupFile = join(REPO_ROOT, 'sdks', sdkPath, `setup${ext}`); - - if (existsSync(setupFile)) { - // For JS/TS, we can import directly - if (ext === '.ts' || ext === '.js') { - const fileUrl = `file://${setupFile}`; - return await import(fileUrl); - } - // For Python, we'd need to use a different approach (spawn python process) - // This will be handled by the runner - return { __pythonSetup: setupFile }; - } - } - - return {}; -} diff --git a/shared/orchestration/src/reporters/console-printer.ts b/shared/orchestration/src/reporters/console-printer.ts deleted file mode 100644 index 8b28721..0000000 --- a/shared/orchestration/src/reporters/console-printer.ts +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Console Printer - Reads CTRF Report and prints to console - * - * This maintains the exact same output format as before, but now reads from CTRF - * instead of directly from TestResult[]. - */ - -import chalk from 'chalk'; -import type { Report, Test } from 'ctrf'; - -/** - * Print CTRF report to console (same format as original printResults) - */ -export function printCTRFReport(report: Report) { - console.log(chalk.blue.bold('\n📊 Test Results\n')); - - // Group tests by SDK (suite field) - const testsBySDK = new Map(); - for (const test of report.results.tests) { - // suite is string[], take the first element or 'unknown' - const suite = (test.suite && test.suite.length > 0) ? test.suite[0] : 'unknown'; - if (!testsBySDK.has(suite)) { - testsBySDK.set(suite, []); - } - testsBySDK.get(suite)!.push(test); - } - - // Print results for each SDK - for (const [sdkPath, tests] of testsBySDK) { - console.log(chalk.cyan.bold(sdkPath)); - - for (const test of tests) { - const statusIcon = test.status === 'passed' - ? chalk.green('✓') - : chalk.red('✗'); - - const duration = chalk.gray(`(${test.duration}ms)`); - - // Extract case ID from test name (format: "sdk/path :: caseId") - const caseId = test.name.split(' :: ')[1] || test.name; - - console.log(` ${statusIcon} ${caseId} ${duration}`); - - // Print error details if test failed - if (test.status === 'failed' && (test.message || test.trace)) { - // Use trace if available (full error), otherwise use message - const errorText = test.trace || test.message || ''; - const errorLines = errorText.split('\n'); - - for (const line of errorLines) { - // Skip stack trace lines (lines starting with " at ") - if (line.trim().startsWith('at ')) { - continue; - } - - // Lines starting with " " are span-level errors, keep their indentation - // Other lines get standard error indentation - if (line.startsWith(' ')) { - console.log(chalk.red(line)); - } else if (line.trim()) { - console.log(chalk.red(` ${line}`)); - } - } - } - } - console.log(''); - } - - // Print summary - const { passed, failed, tests: total } = report.results.summary; - const duration = report.results.summary.stop - report.results.summary.start; - - console.log(chalk.bold('Summary:')); - console.log(` ${chalk.green(`${passed} passed`)}, ${chalk.red(`${failed} failed`)}, ${total} total`); - console.log(chalk.gray(` Time: ${(duration / 1000).toFixed(2)}s\n`)); - - if (failed === 0) { - console.log(chalk.green.bold('✓ All tests passed!\n')); - } else { - console.log(chalk.red.bold('✗ Some tests failed\n')); - } -} diff --git a/shared/orchestration/src/reporters/ctrf-generator.ts b/shared/orchestration/src/reporters/ctrf-generator.ts deleted file mode 100644 index 52d4033..0000000 --- a/shared/orchestration/src/reporters/ctrf-generator.ts +++ /dev/null @@ -1,91 +0,0 @@ -/** - * CTRF Generator - Converts TestResult[] to CTRF Report format - * - * CTRF (Common Test Report Format) is our single source of truth for test data. - * All reporters (console, HTML, etc.) consume CTRF format. - */ - -import type { Report, Test } from 'ctrf'; -import type { TestResult, SDK } from '../types.js'; -import { writeFile, mkdir } from 'fs/promises'; -import { join } from 'path'; - -/** - * Convert our TestResult[] format to CTRF Report format - */ -export function generateCTRFReport( - results: TestResult[], - sdks: SDK[], - totalDuration: number -): Report { - const now = Date.now(); - const startTime = now - totalDuration; - - // Convert each TestResult to CTRF Test - const tests: Test[] = results.map(result => { - const test: Test = { - name: `${result.sdkPath} :: ${result.caseId}`, - status: result.status, - duration: result.duration, - }; - - // Add optional fields - test.suite = [result.sdkPath]; // suite is string[] in CTRF - test.tags = [ - result.sdkPath.startsWith('js/') ? 'javascript' : 'python', - result.caseId - ]; - - // Add error details if test failed - if (result.error) { - test.message = result.error.message.split('\n')[0]; // First line - test.trace = result.error.stack || result.error.message; - } - - return test; - }); - - // Calculate summary - const summary = { - tests: results.length, - passed: results.filter(r => r.status === 'passed').length, - failed: results.filter(r => r.status === 'failed').length, - pending: 0, - skipped: results.filter(r => r.status === 'skipped').length, - other: 0, - start: startTime, - stop: now - }; - - // Build the CTRF report - const report: Report = { - reportFormat: 'CTRF', - specVersion: '1.0.0', - results: { - tool: { - name: 'sentry-ai-test', - version: '1.0.0' - }, - summary, - tests - } - }; - - return report; -} - -/** - * Write CTRF report to file - */ -export async function writeCTRFFile( - report: Report, - outputDir: string = './test-results' -): Promise { - // Ensure output directory exists - await mkdir(outputDir, { recursive: true }); - - const filePath = join(outputDir, 'ctrf-report.json'); - await writeFile(filePath, JSON.stringify(report, null, 2), 'utf-8'); - - return filePath; -} diff --git a/shared/orchestration/src/reporters/html-generator.ts b/shared/orchestration/src/reporters/html-generator.ts deleted file mode 100644 index 2dd3b63..0000000 --- a/shared/orchestration/src/reporters/html-generator.ts +++ /dev/null @@ -1,394 +0,0 @@ -/** - * HTML Generator - Reads CTRF Report and generates HTML report - * - * Uses htm+vhtml for templating (no build step required) - */ - -import htm from 'htm'; -import vhtml from 'vhtml'; -import type { Report, Test } from 'ctrf'; -import { writeFile, mkdir } from 'fs/promises'; -import { join } from 'path'; - -const html = htm.bind(vhtml); - -/** - * Format duration in milliseconds to human-readable format - */ -function formatDuration(ms: number): string { - if (ms < 1000) return `${ms}ms`; - return `${(ms / 1000).toFixed(2)}s`; -} - -/** - * Get status icon for test result - */ -function getStatusIcon(status: string): string { - switch (status) { - case 'passed': return '✓'; - case 'failed': return '✗'; - case 'skipped': return '○'; - default: return '-'; - } -} - -/** - * Generate summary cards HTML - */ -function SummaryCards({ summary }: { summary: Report['results']['summary'] }) { - const duration = summary.stop - summary.start; - - return html` -
-
-

Total Tests

-

${summary.tests}

-
-
-

✓ Passed

-

${summary.passed}

-
-
-

✗ Failed

-

${summary.failed}

-
-
-

Duration

-

${formatDuration(duration)}

-
-
- `; -} - -/** - * Natural sort comparator that handles numeric prefixes correctly. - * E.g., "1-simple" < "2-multi" < "10-binary" (not lexical "1" < "10" < "2") - */ -function naturalSortCompare(a: string, b: string): number { - // Extract numeric prefix if present (e.g., "10-binary" -> 10) - const aMatch = a.match(/^(\d+)/); - const bMatch = b.match(/^(\d+)/); - - // If both have numeric prefixes, compare numerically - if (aMatch && bMatch) { - const aNum = parseInt(aMatch[1], 10); - const bNum = parseInt(bMatch[1], 10); - if (aNum !== bNum) { - return aNum - bNum; - } - // If numeric prefixes are equal, compare the rest lexically - return a.localeCompare(b); - } - - // If only one has a numeric prefix, it comes first - if (aMatch) return -1; - if (bMatch) return 1; - - // Neither has a numeric prefix, compare lexically - return a.localeCompare(b); -} - -/** - * Build test matrix: SDK × Test Case grid - */ -function TestMatrix({ report }: { report: Report }) { - // Extract unique SDKs and test cases - // suite is string[], take first element - const sdks = [...new Set(report.results.tests.map((t: Test) => - (t.suite && t.suite.length > 0) ? t.suite[0] : 'unknown' - ))].sort(); - const testCases = [...new Set( - report.results.tests.map((t: Test) => t.name.split(' :: ')[1] || t.name) - )].sort(naturalSortCompare); - - // Build lookup map for quick access - const testMap = new Map(); - for (const test of report.results.tests) { - const caseId = test.name.split(' :: ')[1] || test.name; - const suite = (test.suite && test.suite.length > 0) ? test.suite[0] : 'unknown'; - const key = `${suite}::${caseId}`; - testMap.set(key, test); - } - - return html` -

Test Matrix

- - - - - ${testCases.map(caseId => html``)} - - - - ${sdks.map(sdk => html` - - - ${testCases.map(caseId => { - const key = `${sdk}::${caseId}`; - const test = testMap.get(key); - - if (!test) { - return html``; - } - - return html` - - `; - })} - - `)} - -
SDK${caseId}
${sdk}- - ${getStatusIcon(test.status)} -
- `; -} - -/** - * Failed tests details section - */ -function FailedTestsDetails({ tests }: { tests: Test[] }) { - const failedTests = tests.filter(t => t.status === 'failed'); - - if (failedTests.length === 0) { - return html``; - } - - return html` -

Failed Tests Details

- ${failedTests.map(test => { - const caseId = test.name.split(' :: ')[1] || test.name; - - return html` -
- - ✗ - ${(test.suite && test.suite.length > 0) ? test.suite[0] : 'unknown'} :: ${caseId} - (${test.duration}ms) - -
- ${test.trace ? html` -
- Details: - 
${test.trace}
-
- ` : ''} -
-
- `; - })} - `; -} - -/** - * Generate complete HTML report from CTRF report - */ -export function generateHTML(report: Report): string { - const title = 'Sentry AI SDK Test Report'; - - const htmlContent = html` - - - - - - ${title} - - - -
-

${title}

- ${SummaryCards({ summary: report.results.summary })} - ${TestMatrix({ report })} - ${FailedTestsDetails({ tests: report.results.tests })} -
- - - `; - - // vhtml returns mixed content: strings for HTML tags, and arrays for special elements like DOCTYPE - // The structure is typically: ["!DOCTYPE", attrs, "..."] - function flattenToString(value: any): string { - if (typeof value === 'string') { - return value; - } - if (Array.isArray(value)) { - // Check if this is a DOCTYPE declaration - if (value[0] === '!DOCTYPE') { - // DOCTYPE + rest of HTML - return '\n' + value.slice(2).map(flattenToString).join(''); - } - // Regular array, flatten all elements - return value.map(flattenToString).join(''); - } - if (typeof value === 'object' && value !== null) { - // Skip objects (like attributes) - return ''; - } - return String(value); - } - - return flattenToString(htmlContent); -} - -/** - * Write HTML report to file - */ -export async function writeHTMLFile( - html: string, - outputDir: string = './test-results' -): Promise { - // Ensure output directory exists - await mkdir(outputDir, { recursive: true }); - - const filePath = join(outputDir, 'test-report.html'); - await writeFile(filePath, html, 'utf-8'); - - return filePath; -} diff --git a/shared/orchestration/src/reporters/index.ts b/shared/orchestration/src/reporters/index.ts deleted file mode 100644 index 4f1b416..0000000 --- a/shared/orchestration/src/reporters/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -/** - * Reporters - Export all report generators and printers - */ - -export { generateCTRFReport, writeCTRFFile } from './ctrf-generator.js'; -export { printCTRFReport } from './console-printer.js'; -export { generateHTML, writeHTMLFile } from './html-generator.js'; diff --git a/shared/orchestration/src/runner.ts b/shared/orchestration/src/runner.ts deleted file mode 100644 index 67eabf1..0000000 --- a/shared/orchestration/src/runner.ts +++ /dev/null @@ -1,272 +0,0 @@ -/** - * Test runner - executes test cases with lifecycle hooks - */ - -import { spawn } from 'child_process'; -import { dirname, join } from 'path'; -import { existsSync } from 'fs'; -import { fileURLToPath } from 'url'; -import type { SDK, TestCase, TestResult, LifecycleHooks, SDKConfig, RunOptions, LocalSentryOptions } from './types.js'; -import { loadSetupHooks } from './discovery.js'; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); - -/** - * Run a single test case - */ -async function runTestCase(testCase: TestCase, hooks: LifecycleHooks, sdk: SDK, options?: RunOptions): Promise { - const startTime = Date.now(); - - try { - // Determine if this is a JS/TS or Python test - const isPython = testCase.filePath.endsWith('.py'); - - if (isPython) { - // Run Python test using subprocess - await runPythonTest(testCase.filePath, testCase.id, sdk.config, sdk.path, options); - } else { - // Run JS/TS test using subprocess for isolation - await runJavaScriptTest(testCase.filePath, testCase.id, sdk.config, sdk.path); - } - - return { - sdkPath: testCase.sdkPath, - caseId: testCase.id, - status: 'passed', - duration: Date.now() - startTime - }; - } catch (error) { - return { - sdkPath: testCase.sdkPath, - caseId: testCase.id, - status: 'failed', - error: error as Error, - duration: Date.now() - startTime - }; - } -} - -/** - * Run a JavaScript test file - */ -function runJavaScriptTest(filePath: string, caseId: string, config?: SDKConfig, sdkPath?: string): Promise { - return new Promise((resolve, reject) => { - // Get the SDK directory (2 levels up from test file) - const sdkDir = dirname(dirname(filePath)); - - // Use the JavaScript test runner wrapper to handle lifecycle hooks - const runnerScript = join(dirname(__dirname), 'js-test-runner.cjs'); - - // Prepare environment with SDK config and overrides - const env = { ...process.env } as NodeJS.ProcessEnv; - - // Pass the SDK path for display in output - if (sdkPath) { - env.SDK_PATH = sdkPath; - } - - // Pass the entire SDK config (including framework_type) - if (config) { - env.SDK_CONFIG = JSON.stringify(config); - } - - // Pass test case specific overrides - if (config?.overrides && config.overrides[caseId]) { - env.SDK_CONFIG_OVERRIDES = JSON.stringify(config.overrides[caseId]); - } - - // Check if verbose mode is enabled - const isVerbose = process.env.SENTRY_AI_TEST_VERBOSE === 'true'; - - const node = spawn('node', [runnerScript, sdkDir, filePath], { - stdio: ['inherit', 'inherit', isVerbose ? 'inherit' : 'pipe'], // Show stderr directly in verbose mode - cwd: sdkDir, - env // Pass parent env with SDK config - }); - - let stderrData = ''; - - // Only capture stderr if not in verbose mode - if (!isVerbose) { - node.stderr?.on('data', (data) => { - stderrData += data.toString(); - }); - } - - node.on('close', (code) => { - if (code === 0) { - resolve(); - } else { - // In verbose mode, stderr was already shown - just report the exit code - if (isVerbose) { - reject(new Error(`JavaScript test exited with code ${code}`)); - return; - } - - // Extract the error message from stderr (line starting with "✗ Test failed:") - // and collect everything after it until the stack trace (Error:) starts - const lines = stderrData.split('\n'); - let errorMsg = ''; - let capturing = false; - - for (const line of lines) { - if (line.startsWith('✗ Test failed:')) { - errorMsg = line.replace('✗ Test failed: ', ''); - capturing = true; - } else if (capturing && line.startsWith('Error:')) { - // Stop at stack trace - break; - } else if (capturing && line.trim()) { - errorMsg += '\n' + line; - } - } - - if (!errorMsg) { - errorMsg = `JavaScript test exited with code ${code}`; - } - - reject(new Error(errorMsg)); - } - }); - - node.on('error', (err) => { - reject(err); - }); - }); -} - -/** - * Run a Python test file - */ -function runPythonTest(filePath: string, caseId: string, config?: SDKConfig, sdkPath?: string, options?: RunOptions): Promise { - return new Promise((resolve, reject) => { - // Get the SDK directory (2 levels up from test file) - const sdkDir = dirname(dirname(filePath)); - - // Check for venv - const venvPython = `${sdkDir}/.venv/bin/python`; - const pythonCmd = existsSync(venvPython) ? venvPython : 'python3'; - - // Use the Python test runner wrapper to handle lifecycle hooks - const runnerScript = join(dirname(__dirname), 'python-test-runner.py'); - - // Prepare environment with SDK config and overrides - const env = { ...process.env } as NodeJS.ProcessEnv; - - // Pass the SDK path for display in output - if (sdkPath) { - env.SDK_PATH = sdkPath; - } - - // Pass the entire SDK config (including framework_type) - if (config) { - env.SDK_CONFIG = JSON.stringify(config); - } - - // Pass test case specific overrides - if (config?.overrides && config.overrides[caseId]) { - env.SDK_CONFIG_OVERRIDES = JSON.stringify(config.overrides[caseId]); - } - - // Pass local Sentry SDK paths (for informational purposes) - if (options?.localSentryPythonPath) { - env.LOCAL_SENTRY_PYTHON_PATH = options.localSentryPythonPath; - } - if (options?.localSentryJavaScriptPath) { - env.LOCAL_SENTRY_JAVASCRIPT_PATH = options.localSentryJavaScriptPath; - } - - // Check if verbose mode is enabled - const isVerbose = process.env.SENTRY_AI_TEST_VERBOSE === 'true'; - - const python = spawn(pythonCmd, [runnerScript, sdkDir, filePath], { - stdio: ['inherit', 'inherit', isVerbose ? 'inherit' : 'pipe'], // Show stderr directly in verbose mode - cwd: sdkDir, - env // Pass parent env with SDK config - }); - - let stderrData = ''; - - // Only capture stderr if not in verbose mode - if (!isVerbose) { - python.stderr?.on('data', (data) => { - stderrData += data.toString(); - }); - } - - python.on('close', (code) => { - if (code === 0) { - resolve(); - } else { - // In verbose mode, stderr was already shown - just report the exit code - if (isVerbose) { - reject(new Error(`Python test exited with code ${code}`)); - return; - } - - // Extract the error message from stderr (line starting with "✗ Test failed:") - // and collect everything after it until the traceback starts - const lines = stderrData.split('\n'); - let errorMsg = ''; - let capturing = false; - - for (const line of lines) { - if (line.startsWith('✗ Test failed:')) { - errorMsg = line.replace('✗ Test failed: ', ''); - capturing = true; - } else if (capturing && (line.startsWith('Traceback') || line.startsWith(' File'))) { - // Stop at traceback - break; - } else if (capturing && line.trim()) { - errorMsg += '\n' + line; - } - } - - if (!errorMsg) { - errorMsg = `Python test exited with code ${code}`; - } - - reject(new Error(errorMsg)); - } - }); - - python.on('error', (err) => { - reject(err); - }); - }); -} - -/** - * Run all test cases for a single SDK - */ -export async function runSDKTests(sdk: SDK, options?: RunOptions): Promise { - const results: TestResult[] = []; - - // Run each test case in its own subprocess (for isolation) - for (const testCase of sdk.cases) { - const result = await runTestCase(testCase, {}, sdk, options); - results.push(result); - - // Stop on first failure if fail-fast is enabled - if (options?.failFast && result.status === 'failed') { - break; - } - } - - return results; -} - -/** - * Run tests for multiple SDKs - */ -export async function runTests(sdks: SDK[], options?: RunOptions): Promise { - const allResults: TestResult[] = []; - - for (const sdk of sdks) { - const results = await runSDKTests(sdk, options); - allResults.push(...results); - } - - return allResults; -} diff --git a/shared/orchestration/src/setup.ts b/shared/orchestration/src/setup.ts deleted file mode 100644 index 4755128..0000000 --- a/shared/orchestration/src/setup.ts +++ /dev/null @@ -1,415 +0,0 @@ -/** - * Setup command - Install all dependencies across the repository - */ - -import { spawn } from 'child_process'; -import { join, resolve } from 'path'; -import { existsSync, statSync, readFileSync, writeFileSync, unlinkSync } from 'fs'; -import chalk from 'chalk'; -import { discoverSDKs } from './discovery.js'; -import { REPO_ROOT } from './discovery.js'; -import type { SetupOptions, LocalSentryOptions } from './types.js'; - -interface SetupResult { - success: boolean; - sdkPath?: string; - step: string; - error?: string; -} - -/** - * Validate local Sentry Python SDK path - */ -function validateLocalSentrySdkPath(path: string): void { - const absolutePath = resolve(path); - - // Check if path exists - if (!existsSync(absolutePath)) { - throw new Error( - chalk.red(`✗ Local Sentry SDK path does not exist: ${path}`) + - '\n' + - chalk.gray(' Check the path and try again.') - ); - } - - // Check if it's a directory - const stats = statSync(absolutePath); - if (!stats.isDirectory()) { - throw new Error( - chalk.red(`✗ Local Sentry SDK path is not a directory: ${path}`) + - '\n' + - chalk.gray(' Path must point to the repository root directory.') - ); - } - - // Check if setup.py exists - const setupPyPath = join(absolutePath, 'setup.py'); - if (!existsSync(setupPyPath)) { - throw new Error( - chalk.red(`✗ Local Sentry SDK path missing setup.py: ${path}`) + - '\n' + - chalk.gray(' Path must be a valid Python package with setup.py.') - ); - } - - // Check if sentry_sdk/ directory exists - const sentrySdkDir = join(absolutePath, 'sentry_sdk'); - if (!existsSync(sentrySdkDir) || !statSync(sentrySdkDir).isDirectory()) { - throw new Error( - chalk.red(`✗ Local Sentry SDK path missing sentry_sdk/ directory: ${path}`) + - '\n' + - chalk.gray(' Path must contain the sentry_sdk package.') - ); - } -} - -/** - * Validate local Sentry JavaScript SDK path - */ -function validateLocalSentryJsSdkPath(path: string): void { - const absolutePath = resolve(path); - - // Check if path exists - if (!existsSync(absolutePath)) { - throw new Error( - chalk.red(`✗ Local Sentry JavaScript SDK path does not exist: ${path}`) + - '\n' + - chalk.gray(' Check the path and try again.') - ); - } - - // Check if it's a directory - const stats = statSync(absolutePath); - if (!stats.isDirectory()) { - throw new Error( - chalk.red(`✗ Local Sentry JavaScript SDK path is not a directory: ${path}`) + - '\n' + - chalk.gray(' Path must point to the repository root directory.') - ); - } - - // Check if packages/ directory exists (monorepo structure) - const packagesDir = join(absolutePath, 'packages'); - if (!existsSync(packagesDir) || !statSync(packagesDir).isDirectory()) { - throw new Error( - chalk.red(`✗ Local Sentry JavaScript SDK path missing packages/ directory: ${path}`) + - '\n' + - chalk.gray(' Path must be the sentry-javascript monorepo with packages/ directory.') - ); - } - - // Check if package.json exists at root (monorepo root) - const rootPackageJson = join(absolutePath, 'package.json'); - if (!existsSync(rootPackageJson)) { - throw new Error( - chalk.red(`✗ Local Sentry JavaScript SDK path missing root package.json: ${path}`) + - '\n' + - chalk.gray(' Path must be a valid npm workspace/monorepo.') - ); - } -} - -/** - * Run a command and return success/failure - */ -function runCommand(command: string, args: string[], cwd: string): Promise { - return new Promise((resolve, reject) => { - const child = spawn(command, args, { - cwd, - stdio: 'pipe', - env: process.env as NodeJS.ProcessEnv - }); - - let stdout = ''; - let stderr = ''; - - child.stdout?.on('data', (data) => { - stdout += data.toString(); - }); - - child.stderr?.on('data', (data) => { - stderr += data.toString(); - }); - - child.on('close', (code) => { - if (code === 0) { - resolve(); - } else { - reject(new Error(stderr || stdout || `Command failed with exit code ${code}`)); - } - }); - - child.on('error', (err) => { - reject(err); - }); - }); -} - -/** - * Setup orchestration dependencies - */ -async function setupOrchestration(): Promise { - const orchestrationDir = join(REPO_ROOT, 'shared', 'orchestration'); - - try { - process.stdout.write(chalk.gray(' Installing dependencies...')); - await runCommand('npm', ['install'], orchestrationDir); - process.stdout.write(chalk.green(' ✓\n')); - - return { - success: true, - step: 'orchestration' - }; - } catch (error) { - process.stdout.write(chalk.red(' ✗\n')); - return { - success: false, - step: 'orchestration', - error: (error as Error).message - }; - } -} - -/** - * Setup a JavaScript SDK - */ -async function setupJavaScriptSDK(sdkPath: string, absolutePath: string, options?: LocalSentryOptions): Promise { - try { - // If local Sentry JavaScript SDK path is provided, link it - if (options?.localSentryJavaScriptPath) { - const absoluteLocalPath = resolve(options.localSentryJavaScriptPath); - validateLocalSentryJsSdkPath(absoluteLocalPath); - - // Read package.json to find which @sentry/* packages are used - const packageJsonPath = join(absolutePath, 'package.json'); - const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf-8')); - const sentryPackages = Object.keys(packageJson.dependencies || {}) - .filter(pkg => pkg.startsWith('@sentry/')); - - if (sentryPackages.length === 0) { - process.stdout.write(chalk.yellow(` ${sdkPath} - No @sentry/* packages found, skipping\n`)); - return { - success: true, - sdkPath, - step: 'npm install (no sentry packages)' - }; - } - - // Link each Sentry package from the local SDK - for (const sentryPkg of sentryPackages) { - process.stdout.write(chalk.gray(` ${sdkPath} - Linking ${sentryPkg}...`)); - - // Get the package name without scope (e.g., @sentry/node -> node) - const packageName = sentryPkg.split('/')[1]; - const packagePath = join(absoluteLocalPath, 'packages', packageName); - - // Check if package exists in local SDK - if (!existsSync(packagePath)) { - process.stdout.write(chalk.yellow(` (not found in local SDK, using npm)\n`)); - continue; - } - - // Link the package: npm link /path/to/sentry-javascript/packages/node - await runCommand('npm', ['link', packagePath], absolutePath); - process.stdout.write(chalk.green(' ✓\n')); - } - - // Install other dependencies - process.stdout.write(chalk.gray(` ${sdkPath} - Installing other dependencies...`)); - await runCommand('npm', ['install'], absolutePath); - process.stdout.write(chalk.green(' ✓\n')); - - return { - success: true, - sdkPath, - step: 'npm install (linked)' - }; - } else { - // Standard install - process.stdout.write(chalk.gray(` ${sdkPath} - Installing dependencies...`)); - await runCommand('npm', ['install'], absolutePath); - process.stdout.write(chalk.green(' ✓\n')); - - return { - success: true, - sdkPath, - step: 'npm install' - }; - } - } catch (error) { - process.stdout.write(chalk.red(' ✗\n')); - return { - success: false, - sdkPath, - step: 'npm install', - error: (error as Error).message - }; - } -} - -/** - * Setup a Python SDK - */ -async function setupPythonSDK(sdkPath: string, absolutePath: string, options?: LocalSentryOptions): Promise { - const results: SetupResult[] = []; - const venvPath = join(absolutePath, '.venv'); - const requirementsPath = join(absolutePath, 'requirements.txt'); - - // Check if requirements.txt exists - if (!existsSync(requirementsPath)) { - process.stdout.write(chalk.yellow(` ${sdkPath} - No requirements.txt found, skipping\n`)); - return results; - } - - // Create venv if it doesn't exist - if (!existsSync(venvPath)) { - try { - process.stdout.write(chalk.gray(` ${sdkPath} - Creating venv...`)); - await runCommand('python3', ['-m', 'venv', '.venv'], absolutePath); - process.stdout.write(chalk.green(' ✓\n')); - - results.push({ - success: true, - sdkPath, - step: 'create venv' - }); - } catch (error) { - process.stdout.write(chalk.red(' ✗\n')); - results.push({ - success: false, - sdkPath, - step: 'create venv', - error: (error as Error).message - }); - return results; // Can't continue without venv - } - } - - // Install requirements - try { - const pipPath = join(venvPath, 'bin', 'pip'); - - // If local Sentry SDK path is provided, install it as editable - if (options?.localSentryPythonPath) { - // Validate the local path - const absoluteLocalPath = resolve(options.localSentryPythonPath); - validateLocalSentrySdkPath(absoluteLocalPath); - - // Install editable Sentry SDK first - process.stdout.write(chalk.gray(` ${sdkPath} - Installing editable Sentry SDK...`)); - await runCommand(pipPath, ['install', '-e', absoluteLocalPath], absolutePath); - process.stdout.write(chalk.green(' ✓\n')); - - // Create temporary requirements.txt without sentry-sdk - const requirementsContent = readFileSync(requirementsPath, 'utf-8'); - const filteredLines = requirementsContent - .split('\n') - .filter(line => !line.trim().startsWith('sentry-sdk')) - .join('\n'); - - const tempRequirementsPath = join(absolutePath, '.requirements-temp.txt'); - writeFileSync(tempRequirementsPath, filteredLines, 'utf-8'); - - // Install other dependencies - process.stdout.write(chalk.gray(` ${sdkPath} - Installing other dependencies...`)); - await runCommand(pipPath, ['install', '-r', '.requirements-temp.txt'], absolutePath); - process.stdout.write(chalk.green(' ✓\n')); - - // Clean up temp file - unlinkSync(tempRequirementsPath); - - results.push({ - success: true, - sdkPath, - step: 'pip install (editable)' - }); - } else { - // Standard install from requirements.txt - process.stdout.write(chalk.gray(` ${sdkPath} - Installing dependencies...`)); - await runCommand(pipPath, ['install', '-r', 'requirements.txt'], absolutePath); - process.stdout.write(chalk.green(' ✓\n')); - - results.push({ - success: true, - sdkPath, - step: 'pip install' - }); - } - } catch (error) { - process.stdout.write(chalk.red(' ✗\n')); - results.push({ - success: false, - sdkPath, - step: 'pip install', - error: (error as Error).message - }); - } - - return results; -} - -/** - * Check if we should setup SDKs for a given language - */ -function shouldSetupLanguage(language: 'js' | 'py', options?: SetupOptions): boolean { - return !options?.language || options.language === language; -} - -/** - * Main setup function - */ -export async function setup(options?: SetupOptions): Promise { - console.log(chalk.blue.bold('\n🔧 Setting up Sentry AI SDK Test Repository\n')); - - const allResults: SetupResult[] = []; - - // Setup orchestration - console.log(chalk.bold('Orchestration')); - const orchestrationResult = await setupOrchestration(); - allResults.push(orchestrationResult); - console.log(''); - - // Discover all SDKs - const sdks = await discoverSDKs(); - const jsSDKs = sdks.filter(sdk => sdk.language === 'js'); - const pySDKs = sdks.filter(sdk => sdk.language === 'py'); - - // Setup JavaScript SDKs - if (shouldSetupLanguage('js', options) && jsSDKs.length > 0) { - console.log(chalk.bold('JavaScript SDKs')); - for (const sdk of jsSDKs) { - const result = await setupJavaScriptSDK(sdk.path, sdk.absolutePath, options); - allResults.push(result); - } - console.log(''); - } - - // Setup Python SDKs - if (shouldSetupLanguage('py', options) && pySDKs.length > 0) { - console.log(chalk.bold('Python SDKs')); - for (const sdk of pySDKs) { - const results = await setupPythonSDK(sdk.path, sdk.absolutePath, options); - allResults.push(...results); - } - console.log(''); - } - - // Print summary - const successful = allResults.filter(r => r.success).length; - const failed = allResults.filter(r => !r.success); - - if (failed.length === 0) { - console.log(chalk.green.bold(`✓ Setup complete! ${successful} steps successful\n`)); - } else { - console.log(chalk.yellow.bold(`⚠ Setup complete with ${failed.length} error(s)\n`)); - console.log(chalk.bold('Failed steps:')); - for (const result of failed) { - const location = result.sdkPath || result.step; - console.log(chalk.red(` ✗ ${location} (${result.step})`)); - if (result.error) { - console.log(chalk.gray(` ${result.error.split('\n')[0]}`)); - } - } - console.log(''); - } -} diff --git a/shared/orchestration/src/types.ts b/shared/orchestration/src/types.ts deleted file mode 100644 index d25909a..0000000 --- a/shared/orchestration/src/types.ts +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Type definitions for the Sentry AI SDK test orchestration system - */ - -export interface TestCase { - id: string; // e.g., "G1", "S1", "A1" - filePath: string; // Absolute path to the test file - sdkPath: string; // e.g., "js/openai" -} - -export interface SDKConfig { - sdk_name: string; - framework_type: 'agentic' | 'low-level'; - overrides?: Record>; // Per-test-case overrides - metadata?: { - sdk_version?: string; - description?: string; - notes?: string; - }; -} - -export interface SDK { - language: 'js' | 'py'; - name: string; // e.g., "openai", "langchain" - path: string; // e.g., "js/openai" - absolutePath: string; // Full file system path - cases: TestCase[]; - hasSetup: boolean; // Whether setup.ts/setup.py exists - config?: SDKConfig; // SDK configuration (if config.json exists) -} - -export interface LifecycleHooks { - beforeAll?: () => Promise | void; - beforeEach?: () => Promise | void; - afterEach?: () => Promise | void; - afterAll?: () => Promise | void; -} - -export interface TestCaseModule { - default: () => Promise | void; -} - -export interface LocalSentryOptions { - localSentryPythonPath?: string; // Path to local Sentry Python SDK (sentry-python) - localSentryJavaScriptPath?: string; // Path to local Sentry JavaScript SDK (sentry-javascript) -} - -export interface SetupOptions extends LocalSentryOptions { - language?: 'js' | 'py'; // Filter by language (e.g., "js" or "py") -} - -export interface RunOptions extends LocalSentryOptions { - sdk?: string; // Filter by SDK (e.g., "js/openai") - case?: string; // Filter by case (e.g., "G1") - all?: boolean; // Run all tests - failFast?: boolean; // Stop SDK tests on first failure -} - -export interface TestResult { - sdkPath: string; - caseId: string; - status: 'passed' | 'failed' | 'skipped'; - error?: Error; - duration: number; // in milliseconds -} diff --git a/shared/orchestration/src/upgrade.ts b/shared/orchestration/src/upgrade.ts deleted file mode 100644 index b174a8b..0000000 --- a/shared/orchestration/src/upgrade.ts +++ /dev/null @@ -1,520 +0,0 @@ -/** - * Upgrade command - Upgrade a package across all SDKs - */ - -import { spawn } from 'child_process'; -import { readFileSync, writeFileSync, existsSync, lstatSync } from 'fs'; -import { join } from 'path'; -import chalk from 'chalk'; -import { discoverSDKs } from './discovery.js'; - -interface UpgradeResult { - success: boolean; - sdkPath: string; - step: 'update' | 'install'; - error?: string; -} - -/** - * Run a command and return success/failure - */ -function runCommand(command: string, args: string[], cwd: string): Promise { - return new Promise((resolve, reject) => { - const child = spawn(command, args, { - cwd, - stdio: 'pipe', - env: process.env as NodeJS.ProcessEnv - }); - - let stdout = ''; - let stderr = ''; - - child.stdout?.on('data', (data) => { - stdout += data.toString(); - }); - - child.stderr?.on('data', (data) => { - stderr += data.toString(); - }); - - child.on('close', (code) => { - if (code === 0) { - resolve(); - } else { - reject(new Error(stderr || stdout || `Command failed with exit code ${code}`)); - } - }); - - child.on('error', (err) => { - reject(err); - }); - }); -} - -/** - * Detect if package is JavaScript or Python based on name - */ -function detectPackageType(packageName: string): 'js' | 'py' { - // JS packages typically: - // - Start with @ (scoped packages like @sentry/node) - // - Are common JS packages (openai, dotenv, etc.) - - if (packageName.startsWith('@')) { - return 'js'; - } - - // Python packages typically use hyphens and don't start with @ - // Common Python packages: sentry-sdk, python-dotenv, openai (can be both!) - - // For ambiguous cases like "openai", we'll need to check what SDKs actually use - // For now, assume JS if starts with @, otherwise Python - // User can disambiguate by checking which SDKs get found - - return 'py'; -} - -/** - * Check if a JavaScript package version exists on npm - */ -async function checkNpmVersionExists(packageName: string, version: string): Promise { - try { - await runCommand('npm', ['view', `${packageName}@${version}`, 'version'], process.cwd()); - return true; - } catch (error) { - return false; - } -} - -/** - * Check if a Python package version exists on PyPI using the JSON API - */ -async function checkPyPIVersionExists(packageName: string, version: string): Promise { - try { - // Use PyPI JSON API to check versions - // This is more reliable than pip index and doesn't require pip to be installed - const https = await import('https'); - - return new Promise((resolve, reject) => { - const url = `https://pypi.org/pypi/${packageName}/json`; - - https.get(url, (res) => { - let data = ''; - - res.on('data', (chunk) => { - data += chunk; - }); - - res.on('end', () => { - try { - if (res.statusCode === 404) { - // Package doesn't exist - resolve(false); - return; - } - - if (res.statusCode !== 200) { - // API error, allow to proceed (install will catch it) - resolve(true); - return; - } - - const json = JSON.parse(data); - const versions = Object.keys(json.releases || {}); - resolve(versions.includes(version)); - } catch (error) { - // Parse error, allow to proceed - resolve(true); - } - }); - }).on('error', (err) => { - // Network error, allow to proceed (install will catch it) - resolve(true); - }); - }); - } catch (error) { - // If check fails, allow to proceed (install will catch the error) - return true; - } -} - -/** - * Update package version in package.json - */ -function updatePackageJson(filePath: string, packageName: string, version: string): boolean { - try { - const content = readFileSync(filePath, 'utf-8'); - const pkg = JSON.parse(content); - - // Check if package exists in dependencies - if (!pkg.dependencies || !pkg.dependencies[packageName]) { - return false; // Package not found - } - - // Update version (exact version, no ^ or ~) - pkg.dependencies[packageName] = version; - - // Write back with pretty formatting - writeFileSync(filePath, JSON.stringify(pkg, null, 2) + '\n', 'utf-8'); - return true; - } catch (error) { - throw new Error(`Failed to update package.json: ${(error as Error).message}`); - } -} - -/** - * Check if a package is linked using npm link - */ -function isNpmLinked(sdkPath: string, packageName: string): boolean { - try { - const nodeModulesPath = join(sdkPath, 'node_modules', packageName); - - // Check if the package exists in node_modules - if (!existsSync(nodeModulesPath)) { - return false; - } - - // Check if it's a symlink (npm link creates symlinks) - const stats = lstatSync(nodeModulesPath); - return stats.isSymbolicLink(); - } catch (error) { - // If we can't determine, assume it's not linked - return false; - } -} - -/** - * Check if a package is installed as editable in a Python venv - */ -async function isEditableInstall(venvPath: string, packageName: string): Promise { - try { - const pipPath = join(venvPath, 'bin', 'pip'); - - // Run pip list --format=json to get package information - const output = await new Promise((resolve, reject) => { - const child = spawn(pipPath, ['list', '--format=json'], { - stdio: 'pipe' - }); - - let stdout = ''; - let stderr = ''; - - child.stdout?.on('data', (data) => { - stdout += data.toString(); - }); - - child.stderr?.on('data', (data) => { - stderr += data.toString(); - }); - - child.on('close', (code) => { - if (code === 0) { - resolve(stdout); - } else { - reject(new Error(stderr || 'pip list failed')); - } - }); - - child.on('error', (err) => { - reject(err); - }); - }); - - // Parse JSON output - const packages = JSON.parse(output); - - // Find the package - const pkg = packages.find((p: any) => p.name === packageName); - - if (!pkg) { - return false; - } - - // Check if it's editable (location contains a path, not site-packages) - // Editable packages have an 'editable_project_location' field or - // their location doesn't point to site-packages - if (pkg.editable_project_location) { - return true; - } - - // Fallback: check if location looks like an editable install - // (contains a path that's not in site-packages) - if (pkg.location && !pkg.location.includes('site-packages')) { - return true; - } - - return false; - } catch (error) { - // If we can't determine, assume it's not editable - return false; - } -} - -/** - * Update package version in requirements.txt - */ -function updateRequirementsTxt(filePath: string, packageName: string, version: string): boolean { - try { - const content = readFileSync(filePath, 'utf-8'); - const lines = content.split('\n'); - - let found = false; - const updatedLines = lines.map(line => { - // Match lines like "sentry-sdk==2.0.0" or "sentry-sdk>=2.0.0" - const match = line.match(/^([a-zA-Z0-9_-]+)(==|>=|<=|>|<)(.+)$/); - if (match && match[1] === packageName) { - found = true; - return `${packageName}==${version}`; - } - return line; - }); - - if (!found) { - return false; // Package not found - } - - writeFileSync(filePath, updatedLines.join('\n'), 'utf-8'); - return true; - } catch (error) { - throw new Error(`Failed to update requirements.txt: ${(error as Error).message}`); - } -} - -/** - * Upgrade JavaScript SDKs - */ -async function upgradeJavaScriptSDKs(packageName: string, version: string): Promise { - const results: UpgradeResult[] = []; - const sdks = await discoverSDKs(); - const jsSDKs = sdks.filter(sdk => sdk.language === 'js'); - - for (const sdk of jsSDKs) { - const packageJsonPath = join(sdk.absolutePath, 'package.json'); - - if (!existsSync(packageJsonPath)) { - continue; - } - - // Check if package is linked via npm link - const isLinked = isNpmLinked(sdk.absolutePath, packageName); - if (isLinked) { - process.stdout.write(chalk.yellow(` ${sdk.path} - Skipping (npm linked)\n`)); - process.stdout.write(chalk.gray(` To upgrade, first unlink:\n`)); - process.stdout.write(chalk.gray(` cd ${sdk.path} && npm unlink ${packageName}\n`)); - process.stdout.write(chalk.gray(` Then run: npm run cli setup\n`)); - continue; - } - - // Try to update package.json - try { - process.stdout.write(chalk.gray(` ${sdk.path} - Updating package.json...`)); - const wasUpdated = updatePackageJson(packageJsonPath, packageName, version); - - if (!wasUpdated) { - process.stdout.write(chalk.gray(' (not used)\n')); - continue; // Package not in this SDK - } - - process.stdout.write(chalk.green(' ✓\n')); - results.push({ - success: true, - sdkPath: sdk.path, - step: 'update' - }); - } catch (error) { - process.stdout.write(chalk.red(' ✗\n')); - results.push({ - success: false, - sdkPath: sdk.path, - step: 'update', - error: (error as Error).message - }); - continue; // Can't install if update failed - } - - // Run npm install - try { - process.stdout.write(chalk.gray(` ${sdk.path} - Installing dependencies...`)); - await runCommand('npm', ['install'], sdk.absolutePath); - process.stdout.write(chalk.green(' ✓\n')); - results.push({ - success: true, - sdkPath: sdk.path, - step: 'install' - }); - } catch (error) { - process.stdout.write(chalk.red(' ✗\n')); - results.push({ - success: false, - sdkPath: sdk.path, - step: 'install', - error: (error as Error).message - }); - } - } - - return results; -} - -/** - * Upgrade Python SDKs - */ -async function upgradePythonSDKs(packageName: string, version: string): Promise { - const results: UpgradeResult[] = []; - const sdks = await discoverSDKs(); - const pySDKs = sdks.filter(sdk => sdk.language === 'py'); - - for (const sdk of pySDKs) { - const requirementsPath = join(sdk.absolutePath, 'requirements.txt'); - const venvPath = join(sdk.absolutePath, '.venv'); - - if (!existsSync(requirementsPath)) { - continue; - } - - // Check if package is installed as editable - if (existsSync(venvPath)) { - const isEditable = await isEditableInstall(venvPath, packageName); - if (isEditable) { - process.stdout.write(chalk.yellow(` ${sdk.path} - Skipping (editable install active)\n`)); - process.stdout.write(chalk.gray(` To upgrade, first remove editable install:\n`)); - process.stdout.write(chalk.gray(` cd ${sdk.path} && .venv/bin/pip uninstall ${packageName}\n`)); - process.stdout.write(chalk.gray(` Then run: npm run cli setup\n`)); - continue; - } - } - - // Try to update requirements.txt - try { - process.stdout.write(chalk.gray(` ${sdk.path} - Updating requirements.txt...`)); - const wasUpdated = updateRequirementsTxt(requirementsPath, packageName, version); - - if (!wasUpdated) { - process.stdout.write(chalk.gray(' (not used)\n')); - continue; // Package not in this SDK - } - - process.stdout.write(chalk.green(' ✓\n')); - results.push({ - success: true, - sdkPath: sdk.path, - step: 'update' - }); - } catch (error) { - process.stdout.write(chalk.red(' ✗\n')); - results.push({ - success: false, - sdkPath: sdk.path, - step: 'update', - error: (error as Error).message - }); - continue; // Can't install if update failed - } - - // Run pip install - if (!existsSync(venvPath)) { - process.stdout.write(chalk.yellow(` ${sdk.path} - Skipping install (no venv found)\n`)); - continue; - } - - try { - process.stdout.write(chalk.gray(` ${sdk.path} - Installing dependencies...`)); - const pipPath = join(venvPath, 'bin', 'pip'); - await runCommand(pipPath, ['install', '-r', 'requirements.txt'], sdk.absolutePath); - process.stdout.write(chalk.green(' ✓\n')); - results.push({ - success: true, - sdkPath: sdk.path, - step: 'install' - }); - } catch (error) { - process.stdout.write(chalk.red(' ✗\n')); - results.push({ - success: false, - sdkPath: sdk.path, - step: 'install', - error: (error as Error).message - }); - } - } - - return results; -} - -/** - * Main upgrade function - */ -export async function upgrade(packageName: string, version: string): Promise { - console.log(chalk.blue.bold(`\n📦 Upgrading ${packageName} to ${version}\n`)); - - // Detect package type - const packageType = detectPackageType(packageName); - - // Check if version exists - if (packageType === 'js') { - console.log(chalk.gray(`Detected as JavaScript package`)); - process.stdout.write(chalk.gray(`Checking if version exists on npm...`)); - - const exists = await checkNpmVersionExists(packageName, version); - if (!exists) { - process.stdout.write(chalk.red(' ✗\n\n')); - console.log(chalk.red.bold(`✗ Version ${version} does not exist for ${packageName}\n`)); - console.log(chalk.gray(`Check available versions with: npm view ${packageName} versions\n`)); - process.exit(1); - } - - process.stdout.write(chalk.green(' ✓\n\n')); - } else { - console.log(chalk.gray(`Detected as Python package`)); - process.stdout.write(chalk.gray(`Checking if version exists on PyPI...`)); - - const exists = await checkPyPIVersionExists(packageName, version); - if (!exists) { - process.stdout.write(chalk.red(' ✗\n\n')); - console.log(chalk.red.bold(`✗ Version ${version} does not exist for ${packageName}\n`)); - console.log(chalk.gray(`Check available versions with: pip index versions ${packageName}\n`)); - process.exit(1); - } - - process.stdout.write(chalk.green(' ✓\n\n')); - } - - let results: UpgradeResult[]; - - if (packageType === 'js') { - results = await upgradeJavaScriptSDKs(packageName, version); - } else { - results = await upgradePythonSDKs(packageName, version); - } - - console.log(''); - - // Filter to only SDKs that were actually updated - const updatedSDKs = new Set(); - for (const result of results) { - if (result.step === 'update' && result.success) { - updatedSDKs.add(result.sdkPath); - } - } - - if (updatedSDKs.size === 0) { - console.log(chalk.yellow(`⚠ No SDKs use ${packageName}\n`)); - return; - } - - // Print summary - const failed = results.filter(r => !r.success); - - if (failed.length === 0) { - console.log(chalk.green.bold(`✓ Upgraded ${updatedSDKs.size} SDK(s) successfully\n`)); - } else { - console.log(chalk.yellow.bold(`⚠ Upgraded with ${failed.length} error(s)\n`)); - console.log(chalk.bold('Failed steps:')); - for (const result of failed) { - console.log(chalk.red(` ✗ ${result.sdkPath} (${result.step})`)); - if (result.error) { - console.log(chalk.gray(` ${result.error.split('\n')[0]}`)); - } - } - console.log(''); - } -} diff --git a/shared/specs/1-simple/fixture-agentic.json b/shared/specs/1-simple/fixture-agentic.json deleted file mode 100644 index f99da46..0000000 --- a/shared/specs/1-simple/fixture-agentic.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "spec_id": "1-simple", - "name": "Basic Completion", - "description": "Single prompt with system message - most basic LLM interaction", - "inputs": { - "model": "gpt-5-nano", - "system": "You are a helpful math assistant. Your anwser only with the final result.", - "prompt": "What is 69 + 96?" - }, - "expectations": { - "spans": { - "min_count": 2, - "items": [ - { - "$ref": "common-spans#/agent_invoke_agent" - }, - { - "$ref": "common-spans#/agent_llm_call", - "parent": "invoke_agent" - } - ] - }, - "events": { - "error_count": 0 - } - } -} diff --git a/shared/specs/1-simple/fixture-low-level.json b/shared/specs/1-simple/fixture-low-level.json deleted file mode 100644 index acbad15..0000000 --- a/shared/specs/1-simple/fixture-low-level.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "spec_id": "1-simple", - "name": "Basic Completion", - "description": "Single prompt with system message - most basic LLM interaction", - "inputs": { - "model": "gpt-5-nano", - "system": "You are a helpful math assistant. Your anwser only with the final result.", - "prompt": "What is 69 + 96?" - }, - "expectations": { - "spans": { - "min_count": 1, - "items": [ - { - "$ref": "common-spans#/llm_call" - } - ] - }, - "events": { - "error_count": 0 - } - } -} diff --git a/shared/specs/1-simple/spec.md b/shared/specs/1-simple/spec.md deleted file mode 100644 index 7106866..0000000 --- a/shared/specs/1-simple/spec.md +++ /dev/null @@ -1,69 +0,0 @@ -# 1-simple: Basic Completion - -## Level & Category -- **Level**: 1 (Basic) -- **Category**: Generation - -## Description -Single prompt with system message. The most basic test to verify Sentry captures a simple LLM interaction. - -## Requirements -- One API call to LLM -- System message + user message -- Get response -- Verify Sentry captures the interaction - -## Example Implementation - -### JavaScript -```javascript -// System message -"You are a helpful math assistant." - -// User message -"What is 69 + 96?" - -// Expected: Response like "165" or "69 + 96 = 165" -``` - -### Python -```python -# System message -"You are a helpful math assistant." - -# User message -"What is 69 + 96?" - -# Expected: Response like "165" or "69 + 96 = 165" -``` - -## Expected Sentry Data - -### Spans -- One span for the LLM API call -- Span should include timing information (start time, duration) -- Span operation should indicate AI/LLM operation - -### Events -- Transaction containing the LLM span -- No error events (successful completion) - -### Metadata -- **Model name**: The specific model used (e.g., "gpt-4", "claude-3-opus") -- **Token counts**: - - Input tokens (prompt tokens) - - Output tokens (completion tokens) - - Total tokens -- **Messages**: - - System message: "You are a helpful math assistant." - - User message: "What is 69 + 96?" - - Assistant response captured -- **Provider**: AI provider name (e.g., "openai", "anthropic") - -### Success Criteria -- ✅ Single LLM call completes successfully -- ✅ Response is received -- ✅ Sentry captures span/transaction for the call -- ✅ Both system and user messages are captured in metadata -- ✅ Token counts are present and accurate -- ✅ Model name is captured diff --git a/shared/specs/10-binary-content-redaction/fixture-agentic.json b/shared/specs/10-binary-content-redaction/fixture-agentic.json deleted file mode 100644 index 97fe714..0000000 --- a/shared/specs/10-binary-content-redaction/fixture-agentic.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "spec_id": "10-binary-content-redaction", - "name": "Binary Content Redaction", - "description": "Tests that binary data (like images) is redacted with '[Blob substitute]' marker in captured spans", - "inputs": { - "model": "gpt-5-nano", - "image_type": "png" - }, - "expectations": { - "spans": { - "min_count": 2, - "items": [ - { - "id": "invoke_agent", - "op": "gen_ai.invoke_agent", - "required_attributes": { - "gen_ai.operation.name": true, - "gen_ai.request.model": "gpt-5-nano", - "gen_ai.usage.input_tokens": true, - "gen_ai.usage.output_tokens": true - } - }, - { - "id": "llm_call", - "op": { - "pattern": "gen_ai.*", - "not": [ - "gen_ai.invoke_agent", - "gen_ai.create_agent", - "gen_ai.execute_tool" - ] - }, - "parent": "invoke_agent", - "required_attributes": { - "gen_ai.operation.name": true, - "gen_ai.request.model": "gpt-5-nano", - "gen_ai.usage.input_tokens": true, - "gen_ai.usage.output_tokens": true, - "gen_ai.request.messages": { - "type": "json_array", - "min_length": 1, - "contains": "[Blob substitute]" - } - } - } - ] - }, - "events": { - "error_count": 0 - } - } -} diff --git a/shared/specs/10-binary-content-redaction/fixture-low-level.json b/shared/specs/10-binary-content-redaction/fixture-low-level.json deleted file mode 100644 index bd1373c..0000000 --- a/shared/specs/10-binary-content-redaction/fixture-low-level.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "spec_id": "10-binary-content-redaction", - "name": "Binary Content Redaction", - "description": "Tests that binary data (like images) is redacted with '[Blob substitute]' marker in captured spans", - "inputs": { - "model": "gpt-5-nano", - "image_type": "png" - }, - "expectations": { - "spans": { - "min_count": 1, - "items": [ - { - "id": "llm_call", - "op": { - "pattern": "gen_ai.*", - "not": [ - "gen_ai.invoke_agent", - "gen_ai.create_agent", - "gen_ai.execute_tool" - ] - }, - "required_attributes": { - "gen_ai.operation.name": true, - "gen_ai.request.model": "gpt-5-nano", - "gen_ai.usage.input_tokens": true, - "gen_ai.usage.output_tokens": true, - "gen_ai.request.messages": { - "type": "json_array", - "min_length": 1, - "contains": "[Blob substitute]" - } - } - } - ] - }, - "events": { - "error_count": 0 - } - } -} diff --git a/shared/specs/10-binary-content-redaction/spec.md b/shared/specs/10-binary-content-redaction/spec.md deleted file mode 100644 index 3a0d00e..0000000 --- a/shared/specs/10-binary-content-redaction/spec.md +++ /dev/null @@ -1,28 +0,0 @@ -# Test 10: Binary Content Redaction - -## Overview - -Tests that when binary data (such as images) is sent to an LLM, Sentry correctly redacts the binary content in the captured span data and replaces it with a substitute marker. - -## Scenario - -1. Send a message containing binary image data to the LLM -2. Verify that the `gen_ai.request.messages` contains the redaction marker "[Blob substitute]" instead of the raw binary data - -## Purpose - -This test verifies that Sentry's AI SDK integration properly handles binary content for telemetry purposes, ensuring: -- Binary data is not sent as raw bytes to Sentry (which would be inefficient and potentially problematic) -- A clear marker indicates that binary content was redacted -- The message structure is preserved with the redaction marker in place - -## Expected Behavior - -- Span should have `gen_ai.request.messages` as a JSON array -- The stringified JSON should contain "[Blob substitute]" indicating binary content was redacted -- The LLM call should complete successfully - -## Test Inputs - -- **model**: The LLM model to use (must support vision/image input) -- **image_type**: Type of image to simulate (default: "png") diff --git a/shared/specs/2-multi-step/fixture-agentic.json b/shared/specs/2-multi-step/fixture-agentic.json deleted file mode 100644 index c10f129..0000000 --- a/shared/specs/2-multi-step/fixture-agentic.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "spec_id": "2-multi-step", - "name": "Multi-step Conversation", - "description": "Multiple API calls with conversation history", - "inputs": { - "model": "gpt-5-nano", - "system": "You are a helpful math assistant. Answer only with the final result.", - "first_prompt": "What is 69 + 96?", - "second_prompt": "Multiply it by 3" - }, - "expectations": { - "spans": { - "min_count": 4, - "items": [ - { - "$ref": "common-spans#/agent_invoke_agent", - "id": "invoke_agent_1" - }, - { - "$ref": "common-spans#/agent_llm_call", - "id": "llm_call_1", - "parent": "invoke_agent_1" - }, - { - "$ref": "common-spans#/agent_invoke_agent", - "id": "invoke_agent_2" - }, - { - "$ref": "common-spans#/agent_llm_call", - "id": "llm_call_2", - "parent": "invoke_agent_2" - } - ] - }, - "events": { - "error_count": 0 - } - } -} diff --git a/shared/specs/2-multi-step/fixture-low-level.json b/shared/specs/2-multi-step/fixture-low-level.json deleted file mode 100644 index 2e346b1..0000000 --- a/shared/specs/2-multi-step/fixture-low-level.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "spec_id": "2-multi-step", - "name": "Multi-step Conversation", - "description": "Multiple API calls with conversation history", - "inputs": { - "model": "gpt-5-nano", - "system": "You are a helpful math assistant. Answer only with the final result.", - "first_prompt": "What is 69 + 96?", - "second_prompt": "Multiply it by 3" - }, - "expectations": { - "spans": { - "min_count": 2, - "items": [ - { - "$ref": "common-spans#/llm_call", - "id": "llm_call_1" - }, - { - "$ref": "common-spans#/llm_call", - "id": "llm_call_2" - } - ] - }, - "events": { - "error_count": 0 - } - } -} diff --git a/shared/specs/2-multi-step/spec.md b/shared/specs/2-multi-step/spec.md deleted file mode 100644 index 7e19af4..0000000 --- a/shared/specs/2-multi-step/spec.md +++ /dev/null @@ -1,97 +0,0 @@ -# 2-multi-step: Multi-step Conversation - -## Level & Category -- **Level**: 2 (Intermediate) -- **Category**: Generation - -## Description -Multiple messages in conversation, multiple API calls. Builds on 1-simple by adding conversation history. Tests that Sentry captures all interactions in a multi-step conversation. - -## Requirements -- First message: same as G1 (add 69+96) -- Second message: "multiply it by 3" -- Two separate API calls -- Verify Sentry captures both interactions - -## Example Implementation - -### JavaScript -```javascript -// System message (for both calls) -"You are a helpful math assistant." - -// First call -const firstResponse = await llm.chat("What is 69 + 96?"); -// Expected: "165" - -// Second call with conversation history -const messages = [ - { role: "user", content: "What is 69 + 96?" }, - { role: "assistant", content: firstResponse }, - { role: "user", content: "Multiply it by 3" } -]; -const secondResponse = await llm.chat(messages); -// Expected: "495" or "165 * 3 = 495" -``` - -### Python -```python -# System message (for both calls) -"You are a helpful math assistant." - -# First call -first_response = llm.chat("What is 69 + 96?") -# Expected: "165" - -# Second call with conversation history -messages = [ - {"role": "user", "content": "What is 69 + 96?"}, - {"role": "assistant", "content": first_response}, - {"role": "user", "content": "Multiply it by 3"} -] -second_response = llm.chat(messages) -# Expected: "495" or "165 * 3 = 495" -``` - -## Expected Sentry Data - -### Spans -- Two separate spans, one for each LLM API call -- Each span should have timing information -- Spans should be properly ordered/nested in transaction - -### Events -- Transaction containing both LLM spans -- No error events (successful completion) - -### Metadata - -**First Call Metadata:** -- **Model name**: The specific model used -- **Token counts**: Input, output, and total tokens for first call -- **Messages**: - - System message: "You are a helpful math assistant." - - User message: "What is 69 + 96?" - - Assistant response: "165" -- **Provider**: AI provider name - -**Second Call Metadata:** -- **Model name**: The specific model used -- **Token counts**: Input, output, and total tokens for second call -- **Messages**: - - System message: "You are a helpful math assistant." - - User message: "What is 69 + 96?" - - Assistant message: "165" - - User message: "Multiply it by 3" - - Assistant response: "495" -- **Provider**: AI provider name -- **Conversation history**: Full message history should be captured - -### Success Criteria -- ✅ Both LLM calls complete successfully -- ✅ Two separate spans are captured -- ✅ First span captures initial Q&A -- ✅ Second span captures conversation history + new message -- ✅ Token counts are accurate for both calls -- ✅ All messages in conversation are captured -- ✅ Model name captured for both calls diff --git a/shared/specs/3-agent-success/spec.md b/shared/specs/3-agent-success/spec.md deleted file mode 100644 index 14c5fce..0000000 --- a/shared/specs/3-agent-success/spec.md +++ /dev/null @@ -1,209 +0,0 @@ -# A1: Agentic Workflow - Success - -## Level & Category -- **Level**: 2 (Intermediate) -- **Category**: Agentic - -## Description -Multi-step calculation using tools: (69+96) * 3 / 2. All calls succeed. Tests that Sentry captures complex agentic workflows with multiple tool calls and LLM interactions. - -## Requirements -- Multiple tool calls (at least 3) -- Successful completion of workflow -- Verify Sentry captures entire workflow chain - -## Tool Definitions - -### JavaScript -```javascript -const tools = [ - { - name: "add", - description: "Add two numbers", - parameters: { - type: "object", - properties: { - a: { type: "number", description: "First number" }, - b: { type: "number", description: "Second number" } - }, - required: ["a", "b"] - } - }, - { - name: "multiply", - description: "Multiply two numbers", - parameters: { - type: "object", - properties: { - a: { type: "number", description: "First number" }, - b: { type: "number", description: "Second number" } - }, - required: ["a", "b"] - } - }, - { - name: "divide", - description: "Divide two numbers", - parameters: { - type: "object", - properties: { - a: { type: "number", description: "Numerator" }, - b: { type: "number", description: "Denominator" } - }, - required: ["a", "b"] - } - } -]; - -function add(a, b) { - return { result: a + b }; -} - -function multiply(a, b) { - return { result: a * b }; -} - -function divide(a, b) { - if (b === 0) { - throw new Error("Division by zero"); - } - return { result: a / b }; -} -``` - -### Python -```python -tools = [ - { - "name": "add", - "description": "Add two numbers", - "parameters": { - "type": "object", - "properties": { - "a": {"type": "number", "description": "First number"}, - "b": {"type": "number", "description": "Second number"} - }, - "required": ["a", "b"] - } - }, - { - "name": "multiply", - "description": "Multiply two numbers", - "parameters": { - "type": "object", - "properties": { - "a": {"type": "number", "description": "First number"}, - "b": {"type": "number", "description": "Second number"} - }, - "required": ["a", "b"] - } - }, - { - "name": "divide", - "description": "Divide two numbers", - "parameters": { - "type": "object", - "properties": { - "a": {"type": "number", "description": "Numerator"}, - "b": {"type": "number", "description": "Denominator"} - }, - "required": ["a", "b"] - } - } -] - -def add(a, b): - return {"result": a + b} - -def multiply(a, b): - return {"result": a * b} - -def divide(a, b): - if b == 0: - raise ValueError("Division by zero") - return {"result": a / b} -``` - -## Example Implementation - -### JavaScript -```javascript -// System message -"You are a helpful math assistant. Use the provided tools to perform calculations." - -// Initial prompt -"Calculate (69 + 96) * 3 / 2" - -// Expected flow: -// 1. LLM calls add(69, 96) -// 2. Tool returns: { result: 165 } -// 3. LLM calls multiply(165, 3) -// 4. Tool returns: { result: 495 } -// 5. LLM calls divide(495, 2) -// 6. Tool returns: { result: 247.5 } -// 7. LLM responds: "The result is 247.5" -``` - -### Python -```python -# System message -"You are a helpful math assistant. Use the provided tools to perform calculations." - -# Initial prompt -"Calculate (69 + 96) * 3 / 2" - -# Expected flow: -# 1. LLM calls add(69, 96) -# 2. Tool returns: {"result": 165} -# 3. LLM calls multiply(165, 3) -# 4. Tool returns: {"result": 495} -# 5. LLM calls divide(495, 2) -# 6. Tool returns: {"result": 247.5} -# 7. LLM responds: "The result is 247.5" -``` - -## Expected Sentry Data - -### Spans -- Multiple spans for LLM API calls (may be multiple calls or single call with tools) -- Spans for each tool execution (add, multiply, divide) -- Proper parent-child relationship showing workflow hierarchy -- Each span should have timing information - -### Events -- Transaction containing all workflow spans -- No error events (successful completion) - -### Metadata - -**LLM Call Metadata:** -- **Model name**: The specific model used -- **Token counts**: Total tokens across all LLM interactions -- **Messages**: Initial prompt and any follow-up messages -- **Tools available**: List of tool definitions provided to LLM -- **Provider**: AI provider name - -**Tool Call Metadata (for each tool):** -- **Tool name**: "add", "multiply", "divide" -- **Tool arguments**: - - add(69, 96) - - multiply(165, 3) - - divide(495, 2) -- **Tool results**: - - { result: 165 } - - { result: 495 } - - { result: 247.5 } -- **Execution order**: Clear ordering of tool calls - -**Final Response:** -- Complete answer: "The result is 247.5" - -### Success Criteria -- ✅ All tool calls complete successfully -- ✅ All tool executions are captured as spans -- ✅ LLM interactions are captured -- ✅ Tool arguments and results are captured -- ✅ Workflow chain is clear (parent-child relationships) -- ✅ Token counts are accurate -- ✅ Final result is correct (247.5) -- ✅ All spans properly nested in transaction diff --git a/shared/specs/4-simple-with-error/spec.md b/shared/specs/4-simple-with-error/spec.md deleted file mode 100644 index 6e80d70..0000000 --- a/shared/specs/4-simple-with-error/spec.md +++ /dev/null @@ -1,77 +0,0 @@ -# 4-simple-with-error: Basic Completion with Application Error - -## Level & Category -- **Level**: 1 (Basic) -- **Category**: Generation - -## Description -Application code throws an error after receiving LLM response. Tests that Sentry captures both the successful LLM interaction and the application error with proper context. - -## Requirements -- Start an LLM call (same as G1) -- Throw an exception in application code after receiving response -- Verify Sentry captures both the LLM interaction and the error - -## Example Implementation - -### JavaScript -```javascript -// System message -"You are a helpful math assistant." - -try { - const response = await llm.chat("What is 69 + 96?"); - // Intentionally throw error after receiving response - throw new Error("Application error during processing"); -} catch (error) { - // Sentry should capture this error -} -``` - -### Python -```python -# System message -"You are a helpful math assistant." - -try: - response = llm.chat("What is 69 + 96?") - # Intentionally throw error after receiving response - raise ValueError("Application error during processing") -except Exception as e: - # Sentry should capture this error - pass -``` - -## Expected Sentry Data - -### Spans -- One span for the LLM API call -- Span should complete successfully before error occurs -- Span should be part of the errored transaction - -### Events -- Transaction containing the LLM span (marked as errored) -- Error event with exception details -- Error should be linked to the transaction - -### Metadata - -**LLM Span Metadata:** -- **Model name**: The specific model used -- **Token counts**: Input, output, and total tokens -- **Messages**: System message, user message, and assistant response -- **Provider**: AI provider name - -**Error Event Metadata:** -- **Exception type**: Error/ValueError -- **Exception message**: "Application error during processing" -- **Stack trace**: Full stack trace showing where error occurred -- **Context**: Error occurred after LLM call completed - -### Success Criteria -- ✅ LLM call completes successfully -- ✅ LLM span is captured with all metadata -- ✅ Error event is captured -- ✅ Error is properly linked to transaction containing LLM span -- ✅ Stack trace shows error location -- ✅ Both LLM data and error data are present in Sentry diff --git a/shared/specs/5-streaming/spec.md b/shared/specs/5-streaming/spec.md deleted file mode 100644 index 70569cd..0000000 --- a/shared/specs/5-streaming/spec.md +++ /dev/null @@ -1,86 +0,0 @@ -# S1: Basic Streaming - -## Level & Category -- **Level**: 3 (Advanced) -- **Category**: Streaming - -## Description -Streaming version of G1. Simple streaming completion. Tests that Sentry captures streaming LLM interactions correctly, including the full response assembled from chunks. - -## Requirements -- Stream response from LLM -- Process chunks as they arrive -- Verify Sentry captures streaming interaction -- Verify complete response is captured - -## Example Implementation - -### JavaScript -```javascript -// System message -"You are a helpful math assistant." - -// User message (same as G1, but streaming) -const stream = await llm.chat("What is 69 + 96?", { stream: true }); - -let fullResponse = ""; -for await (const chunk of stream) { - fullResponse += chunk.content; - // Process chunk -} - -// Verify complete response was streamed -// Expected: "165" or "69 + 96 = 165" -``` - -### Python -```python -# System message -"You are a helpful math assistant." - -# User message (same as G1, but streaming) -stream = llm.chat("What is 69 + 96?", stream=True) - -full_response = "" -for chunk in stream: - full_response += chunk.content - # Process chunk - -# Verify complete response was streamed -# Expected: "165" or "69 + 96 = 165" -``` - -## Expected Sentry Data - -### Spans -- One span for the streaming LLM API call -- Span should include timing from start to last chunk -- Span should indicate streaming mode - -### Events -- Transaction containing the streaming LLM span -- No error events (successful completion) - -### Metadata -- **Model name**: The specific model used -- **Token counts**: - - Input tokens (prompt tokens) - - Output tokens (completion tokens) - - Total tokens -- **Messages**: - - System message: "You are a helpful math assistant." - - User message: "What is 69 + 96?" - - Complete assistant response (assembled from all chunks) -- **Provider**: AI provider name -- **Streaming**: Indicator that this was a streaming call -- **Chunks received**: Count or indication of streaming chunks (if supported) - -### Success Criteria -- ✅ Streaming LLM call completes successfully -- ✅ All chunks are received and processed -- ✅ Complete response is assembled correctly -- ✅ Sentry captures span for streaming call -- ✅ Full response is captured (not just final chunk) -- ✅ Token counts reflect complete response -- ✅ Timing captures full stream duration -- ✅ Streaming mode is indicated in metadata diff --git a/shared/specs/6-streaming-with-error/spec.md b/shared/specs/6-streaming-with-error/spec.md deleted file mode 100644 index a209f4e..0000000 --- a/shared/specs/6-streaming-with-error/spec.md +++ /dev/null @@ -1,100 +0,0 @@ -# S2: Streaming with Application Error - -## Level & Category -- **Level**: 3 (Advanced) -- **Category**: Streaming - -## Description -Streaming version of G2. Error occurs during stream processing. Tests that Sentry captures partial streaming data when an error interrupts the stream processing. - -## Requirements -- Start streaming response (same prompt as G1) -- Throw error while processing chunks -- Verify Sentry captures partial stream + error - -## Example Implementation - -### JavaScript -```javascript -// System message -"You are a helpful math assistant." - -const stream = await llm.chat("What is 69 + 96?", { stream: true }); - -let chunkCount = 0; -try { - for await (const chunk of stream) { - chunkCount++; - if (chunkCount > 3) { - throw new Error("Stream processing error"); - } - } -} catch (error) { - // Sentry should capture error with streaming context -} -``` - -### Python -```python -# System message -"You are a helpful math assistant." - -stream = llm.chat("What is 69 + 96?", stream=True) - -chunk_count = 0 -try: - for chunk in stream: - chunk_count += 1 - if chunk_count > 3: - raise RuntimeError("Stream processing error") -except Exception as e: - # Sentry should capture error with streaming context - pass -``` - -## Expected Sentry Data - -### Spans -- One span for the streaming LLM API call -- Span should be marked as errored or interrupted -- Timing should show when stream was interrupted - -### Events -- Transaction containing the streaming span (marked as errored) -- Error event with exception details -- Error should be linked to the streaming transaction - -### Metadata - -**LLM Span Metadata:** -- **Model name**: The specific model used -- **Token counts**: Partial token count (if available) -- **Messages**: - - System message: "You are a helpful math assistant." - - User message: "What is 69 + 96?" - - Partial assistant response (chunks received before error) -- **Provider**: AI provider name -- **Streaming**: Indicator that this was a streaming call -- **Chunks received**: Number of chunks processed before error (3) -- **Stream interrupted**: Indication that stream didn't complete - -**Error Event Metadata:** -- **Exception type**: Error/RuntimeError -- **Exception message**: "Stream processing error" -- **Stack trace**: Full stack trace showing where error occurred -- **Context**: - - Error occurred during streaming - - Number of chunks processed before error - - Partial response data (if available) - -### Success Criteria -- ✅ Streaming starts successfully -- ✅ First 3 chunks are received and processed -- ✅ Error is thrown on 4th chunk -- ✅ Streaming span is captured -- ✅ Partial response data is captured -- ✅ Error event is captured -- ✅ Error is properly linked to streaming transaction -- ✅ Clear indication that stream was interrupted -- ✅ Stack trace shows error location -- ✅ Chunk count or partial data is preserved diff --git a/shared/specs/7-agent-llm-error/spec.md b/shared/specs/7-agent-llm-error/spec.md deleted file mode 100644 index 6eb1ce6..0000000 --- a/shared/specs/7-agent-llm-error/spec.md +++ /dev/null @@ -1,97 +0,0 @@ -# A2: Agentic Workflow - Error During LLM Call - -## Level & Category -- **Level**: 2 (Intermediate) -- **Category**: Agentic - -## Description -Agentic workflow where application error occurs after a tool call. Tests that Sentry captures partial workflow execution plus the error with full context. - -## Requirements -- Start agentic workflow with tools -- First tool call succeeds -- Application error after receiving tool result -- Verify Sentry captures partial workflow + error - -## Tool Definitions -Same tools as A1 (add, multiply, divide, subtract). - -## Example Implementation - -### JavaScript -```javascript -// System message -"You are a helpful math assistant. Use the provided tools to perform calculations." - -try { - // Start calculation - first tool call succeeds - const response = await llm.chat("Calculate 69 + 96", { tools }); - // Tool successfully returns 165 - - // Throw error after receiving result - throw new Error("Error processing calculation result"); -} catch (error) { - // Sentry should capture error with workflow context -} -``` - -### Python -```python -# System message -"You are a helpful math assistant. Use the provided tools to perform calculations." - -try: - # Start calculation - first tool call succeeds - response = llm.chat("Calculate 69 + 96", tools=tools) - # Tool successfully returns 165 - - # Throw error after receiving result - raise RuntimeError("Error processing calculation result") -except Exception as e: - # Sentry should capture error with workflow context - pass -``` - -## Expected Sentry Data - -### Spans -- Span(s) for LLM API call(s) before error -- Span for successful tool execution (add) -- Spans should complete successfully before error -- All spans should be part of the errored transaction - -### Events -- Transaction containing workflow spans (marked as errored) -- Error event with exception details -- Error should be linked to the transaction - -### Metadata - -**LLM Call Metadata:** -- **Model name**: The specific model used -- **Token counts**: Tokens for the call that completed -- **Messages**: Initial prompt -- **Tools available**: List of tool definitions -- **Provider**: AI provider name - -**Tool Call Metadata:** -- **Tool name**: "add" -- **Tool arguments**: add(69, 96) -- **Tool result**: { result: 165 } - -**Error Event Metadata:** -- **Exception type**: Error/RuntimeError -- **Exception message**: "Error processing calculation result" -- **Stack trace**: Full stack trace -- **Context**: Error occurred after tool call completed -- **Workflow state**: Which tools were called before error - -### Success Criteria -- ✅ First tool call (add) completes successfully -- ✅ Tool execution span is captured with all metadata -- ✅ LLM interaction span is captured -- ✅ Error event is captured -- ✅ Error is properly linked to transaction -- ✅ Partial workflow state is preserved in Sentry -- ✅ Stack trace shows error location -- ✅ Clear indication that workflow was interrupted diff --git a/shared/specs/8-agent-tool-error/spec.md b/shared/specs/8-agent-tool-error/spec.md deleted file mode 100644 index 0e82b16..0000000 --- a/shared/specs/8-agent-tool-error/spec.md +++ /dev/null @@ -1,96 +0,0 @@ -# A3: Agentic Workflow - Error During Tool Execution - -## Level & Category -- **Level**: 2 (Intermediate) -- **Category**: Agentic - -## Description -Tool/function execution throws an error (division by zero). Tests that Sentry captures errors that occur within tool execution with full context. - -## Requirements -- LLM calls divide tool -- Tool throws division by zero error -- Workflow handles or propagates error -- Verify Sentry captures tool error with context - -## Tool Definitions -Same tools as A1 (add, multiply, divide, subtract). The divide tool includes error handling for division by zero. - -## Example Implementation - -### JavaScript -```javascript -// System message -"You are a helpful math assistant. Use the provided tools to perform calculations." - -// Prompt that triggers division by zero -"Calculate 165 divided by 0" -// Note: 165 is the sum of 69 + 96 - -// Expected flow: -// 1. LLM calls divide(165, 0) -// 2. Tool throws: Error("Division by zero") -// 3. Sentry captures the tool error with full context -``` - -### Python -```python -# System message -"You are a helpful math assistant. Use the provided tools to perform calculations." - -# Prompt that triggers division by zero -"Calculate 165 divided by 0" -# Note: 165 is the sum of 69 + 96 - -# Expected flow: -# 1. LLM calls divide(165, 0) -# 2. Tool throws: ValueError("Division by zero") -# 3. Sentry captures the tool error with full context -``` - -## Expected Sentry Data - -### Spans -- Span(s) for LLM API call -- Span for tool execution (divide) - marked as errored -- Proper parent-child relationship showing where error occurred - -### Events -- Transaction containing workflow spans (marked as errored) -- Error event with exception details from tool execution -- Error should be linked to the tool execution span - -### Metadata - -**LLM Call Metadata:** -- **Model name**: The specific model used -- **Token counts**: Tokens for the LLM call -- **Messages**: Prompt requesting division by zero -- **Tools available**: List of tool definitions -- **Provider**: AI provider name - -**Tool Call Metadata:** -- **Tool name**: "divide" -- **Tool arguments**: divide(165, 0) -- **Tool error**: Division by zero error - -**Error Event Metadata:** -- **Exception type**: Error/ValueError -- **Exception message**: "Division by zero" -- **Stack trace**: Stack trace showing error in divide function -- **Context**: - - Tool being executed: "divide" - - Arguments: a=165, b=0 - - Error occurred during tool execution -- **Workflow state**: What happened before the tool error - -### Success Criteria -- ✅ LLM interaction is captured -- ✅ Tool call (divide) is captured -- ✅ Tool arguments (165, 0) are captured -- ✅ Error event is captured -- ✅ Error is linked to tool execution span -- ✅ Stack trace shows error in divide function -- ✅ Clear indication that error occurred in tool, not application code -- ✅ Tool execution span is marked as errored -- ✅ Transaction is marked as errored diff --git a/shared/specs/9-message-truncation/fixture-agentic.json b/shared/specs/9-message-truncation/fixture-agentic.json deleted file mode 100644 index ed54cb1..0000000 --- a/shared/specs/9-message-truncation/fixture-agentic.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "spec_id": "9-message-truncation", - "name": "Message Truncation", - "description": "Tests that large message content is truncated while original_length is preserved", - "inputs": { - "model": "gpt-5-nano", - "message_size_kb": 9, - "message_count": 3 - }, - "expectations": { - "spans": { - "min_count": 2, - "items": [ - { - "id": "invoke_agent", - "op": "gen_ai.invoke_agent", - "required_attributes": { - "gen_ai.operation.name": true, - "gen_ai.request.model": "gpt-5-nano", - "gen_ai.usage.input_tokens": true, - "gen_ai.usage.output_tokens": true - } - }, - { - "id": "llm_call", - "op": { - "pattern": "gen_ai.*", - "not": [ - "gen_ai.invoke_agent", - "gen_ai.create_agent", - "gen_ai.execute_tool" - ] - }, - "parent": "invoke_agent", - "required_attributes": { - "gen_ai.operation.name": true, - "gen_ai.request.model": "gpt-5-nano", - "gen_ai.usage.input_tokens": true, - "gen_ai.usage.output_tokens": true, - "gen_ai.request.messages.original_length": true, - "gen_ai.request.messages": { - "type": "json_array", - "min_length": 1, - "length_lte": "gen_ai.request.messages.original_length", - "items_have": ["role", "content"] - } - } - } - ] - }, - "events": { - "error_count": 0 - } - } -} diff --git a/shared/specs/9-message-truncation/fixture-low-level.json b/shared/specs/9-message-truncation/fixture-low-level.json deleted file mode 100644 index 5592c33..0000000 --- a/shared/specs/9-message-truncation/fixture-low-level.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "spec_id": "9-message-truncation", - "name": "Message Truncation", - "description": "Tests that large message content is truncated while original_length is preserved", - "inputs": { - "model": "gpt-5-nano", - "message_size_kb": 9, - "message_count": 3 - }, - "expectations": { - "spans": { - "min_count": 1, - "items": [ - { - "id": "llm_call", - "op": { - "pattern": "gen_ai.*", - "not": [ - "gen_ai.invoke_agent", - "gen_ai.create_agent", - "gen_ai.execute_tool" - ] - }, - "required_attributes": { - "gen_ai.operation.name": true, - "gen_ai.request.model": "gpt-5-nano", - "gen_ai.usage.input_tokens": true, - "gen_ai.usage.output_tokens": true, - "gen_ai.request.messages.original_length": true, - "gen_ai.request.messages": { - "type": "json_array", - "min_length": 1, - "length_lte": "gen_ai.request.messages.original_length", - "items_have": ["role", "content"] - } - } - } - ] - }, - "events": { - "error_count": 0 - } - } -} diff --git a/shared/specs/9-message-truncation/spec.md b/shared/specs/9-message-truncation/spec.md deleted file mode 100644 index 03668c8..0000000 --- a/shared/specs/9-message-truncation/spec.md +++ /dev/null @@ -1,29 +0,0 @@ -# Test 9: Message Truncation - -## Overview - -Tests that when large messages are sent to an LLM, Sentry correctly tracks the original message count vs. the potentially truncated message count in the captured span data. - -## Scenario - -1. Send three large messages (each ~9KB of data) to the LLM -2. Verify that `gen_ai.request.messages` array length is less than or equal to `gen_ai.request.messages.original_length` - -## Purpose - -This test verifies that Sentry's AI SDK integration properly handles message truncation for telemetry purposes, ensuring: -- The original message count is preserved as `gen_ai.request.messages.original_length` -- The actual captured messages may be truncated for telemetry size limits -- The relationship `len(messages) <= original_length` always holds - -## Expected Behavior - -- Span should have `gen_ai.request.messages.original_length` attribute with value 3 (or more, depending on system message handling) -- Span should have `gen_ai.request.messages` as a JSON array -- The array length should be less than or equal to `gen_ai.request.messages.original_length` - -## Test Inputs - -- **model**: The LLM model to use -- **message_size_kb**: Size of each message content in kilobytes (default: 9) -- **message_count**: Number of large messages to send (default: 3) diff --git a/shared/specs/README.md b/shared/specs/README.md deleted file mode 100644 index 38b1033..0000000 --- a/shared/specs/README.md +++ /dev/null @@ -1,460 +0,0 @@ -# Test Specifications & Fixtures - -This directory contains test specifications and fixture expectations for all test scenarios. - -## Directory Structure - -Each test specification has its own directory: - -``` -shared/specs/ -├── 1-simple/ -│ ├── spec.md # Human-readable specification -│ ├── fixture-agentic.json # Expectations for agentic frameworks -│ └── fixture-low-level.json # Expectations for low-level frameworks -├── 2-multi-step/ -│ └── ... -└── ... -``` - -## Current Test Cases - -Test cases are identified by spec ID (e.g., "1-simple", "2-multi-step"). Each has: - -- **JSON fixture(s)** in `shared/specs/{spec-id}/` defining expectations -- **JS implementation(s)** in `sdks/js/*/cases/` -- **Python implementation(s)** in `sdks/py/*/cases/` - -### Implemented - -- **1-simple**: Basic Completion - Single prompt with system message -- **2-multi-step**: Multi-step conversation - Two API calls with conversation history -- **9-message-truncation**: Message Truncation - Tests array length vs original_length constraint -- **10-binary-content-redaction**: Binary Content Redaction - Tests that binary data is redacted with "[Blob substitute]" - -### Planned - -- **3-agent-success**: Agentic workflow - success path -- **4-simple-with-error**: Basic completion with application error -- **5-streaming**: Basic streaming -- **6-streaming-with-error**: Streaming with application error -- **7-agent-llm-error**: Agentic workflow - error during LLM call -- **8-agent-tool-error**: Agentic workflow - error during tool execution - -## Sentry Features to Verify - -Each test must verify that Sentry captures: - -1. **Performance tracing** - Spans and transactions with proper timing -2. **AI monitoring data** - Model name, token counts, prompts, completions -3. **Error tracking** - Exceptions with context and stack traces (for error tests) - -## Framework Types & Fixture Variants - -AI SDKs fall into two categories based on the span hierarchy they produce. - -### Agentic Frameworks - -Frameworks that wrap LLM calls in agent abstraction spans: - -- **Vercel AI SDK** (`js/vercel`) - Produces `gen_ai.invoke_agent` parent spans -- **OpenAI Agents SDK** (`py/openai-agents`) - Produces agent workflow spans - -**Span hierarchy example:** - -``` -gen_ai.invoke_agent (parent) - └─ gen_ai.chat or gen_ai.generate_text (child) -``` - -### Low-Level Frameworks - -Frameworks that directly produce LLM call spans without agent wrappers: - -- **OpenAI SDK** (`js/openai`) - Direct `gen_ai.chat` spans only -- **Anthropic SDK** (`js/anthropic`) - Direct LLM call spans -- **Google GenAI SDK** (`py/google-genai`) - Direct LLM call spans - -**Span hierarchy example:** - -``` -gen_ai.chat (no parent) -``` - -### Using Fixture Variants - -Each test case folder contains multiple fixture files to handle both framework types: - -- `fixture-agentic.json` - Expects agent parent spans + LLM child spans -- `fixture-low-level.json` - Expects only direct LLM call spans - -Framework type is configured per SDK in `config.json`, not in individual test files: - -**SDK Config (config.json):** - -```json -{ - "sdk_name": "vercel", - "framework_type": "agentic", - "overrides": {} -} -``` - -**JavaScript Test Case:** - -```javascript -const { runTestCase } = require("../../_test-utils/test-runner.cjs"); -const { Sentry } = require("../setup"); - -async function testLogic(inputs) { - // Your test logic -} - -// Framework type loaded from config.json automatically -module.exports = runTestCase("1-simple", testLogic, Sentry); -``` - -**Python Test Case:** - -```python -from test_runner import run_test_case - -async def test_logic(inputs): - # Your test logic - pass - -# Framework type loaded from config.json automatically -test_case = run_test_case("1-simple", test_logic) -main = test_case["main"] -assert_sentry = test_case["assert_sentry"] -``` - -**Important:** Each SDK's `config.json` defines its framework type. All test cases in that SDK use the same framework type automatically. - -### SDK Framework Type Mapping - -When adding a new SDK, determine its framework type first, then use the same type across all test cases for that SDK. - -| SDK Path | Framework Type | Reason | -| ------------------ | -------------- | ------------------------------------------- | -| `js/vercel` | `agentic` | Produces `gen_ai.invoke_agent` parent spans | -| `js/openai` | `low-level` | Direct `gen_ai.chat` spans only | -| `js/anthropic` | `low-level` | Direct LLM call spans only | -| `py/openai-agents` | `agentic` | Produces agent workflow spans | -| `py/google-genai` | `low-level` | Direct LLM call spans only | - -**How to determine framework type for a new SDK:** - -1. Run a simple test case with the SDK -2. Examine the captured spans -3. If you see agent/workflow wrapper spans → `agentic` -4. If you only see direct LLM call spans → `low-level` - -## Fixture Format - -Fixtures define expected spans, transactions, and events in a language-agnostic JSON format. - -### Example Fixture - -```json -{ - "spec_id": "1-simple", - "name": "Basic Completion", - "inputs": { - "model": "gpt-5-nano", - "system": "You are a helpful math assistant.", - "prompt": "What is 69 + 96?" - }, - "expectations": { - "spans": { - "min_count": 3, - "items": [ - { - "id": "invoke_agent", - "op": "gen_ai.invoke_agent", - "required_attributes": { - "gen_ai.response.model": "gpt-5-nano", - "gen_ai.response.text": true, - "gen_ai.usage.input_tokens": true - } - }, - { - "id": "generate_text", - "op": ["gen_ai.chat", "gen_ai.generate_text"], - "parent": "invoke_agent", - "required_attributes": { - "gen_ai.request.model": "gpt-5-nano" - } - } - ] - }, - "events": { - "error_count": 0 - } - } -} -``` - -### Fixture Format Specification - -**Top-level fields:** - -- `spec_id` (string): Unique identifier matching directory name -- `name` (string): Human-readable test name -- `description` (string, optional): Detailed description -- `inputs` (object): Test inputs (model, system message, prompt, etc.) -- `expectations` (object): What to verify in captured Sentry data - -**Expectations structure:** - -- `spans` (object): - - - `min_count` (number): Minimum number of spans expected - - `items` (array): Specific spans to verify - -- `spans.items[]` (object): - - - `id` (string): Unique identifier for this span (used for parent references) - - `op` (string | string[]): Operation name(s) to match - - `parent` (string, optional): ID of parent span (verifies hierarchy) - - `required_attributes` (object, optional): Attributes to verify - -- `required_attributes` format: - - - `"attribute.name": true` - Check presence only - - `"attribute.name": "value"` - Check exact match - - `"attribute.name": "pattern*"` - Wildcard pattern matching (see below) - -- `events` (object): - - `error_count` (number): Expected number of error events - -### Key Features - -- `op` can be string or array (matches any of the ops) -- `required_attributes` with `true` = just check presence -- `required_attributes` with value = check exact match or wildcard pattern -- `parent` = verifies span hierarchy -- `min_count` = minimum spans (allows extra spans from SDK) - -### Wildcard Pattern Matching - -Fixture attribute values support wildcard patterns using `*` for flexible matching: - -**Pattern Types:** - -| Pattern | Description | Example | Matches | -| --------- | ----------- | -------------- | ----------------------------- | -| `"foo*"` | Starts with | `"gpt-*"` | `gpt-5-nano`, `gpt-5-nano` | -| `"*foo"` | Ends with | `"*-mini"` | `gpt-5-nano`, `claude-3-mini` | -| `"*foo*"` | Contains | `"*4o*"` | `gpt-5-nano`, `gpt-5-nano` | -| `"foo"` | Exact match | `"gpt-5-nano"` | `gpt-5-nano` only | - -**Use Cases:** - -- **Model versions**: `"gen_ai.request.model": "gpt-4*"` matches any GPT-4 variant -- **Flexible IDs**: `"span_id": "*"` would match any non-empty span ID (but prefer `true` for presence) -- **URL patterns**: `"http.url": "https://api.openai.com/*"` matches any OpenAI API endpoint -- **Token ranges**: Not supported - use `true` for presence checking instead - -**Examples:** - -```json -{ - "required_attributes": { - "gen_ai.request.model": "gpt-*", // Matches gpt-5-nano, gpt-5-nano, etc. - "gen_ai.response.model": "gemini-*", // Matches gemini-2.5-flash-lite, gemini-1.5-pro - "gen_ai.provider": "*anthropic*", // Matches "anthropic", "anthropic-vertex", etc. - "http.url": "https://api.openai.com/*", // Matches any OpenAI API URL - "gen_ai.response.text": true // Just check presence (no pattern needed) - } -} -``` - -**Important Notes:** - -- Wildcards work on string values only (not numbers or booleans) -- Empty wildcards (`"*"`, `"**"`) are invalid and will not match anything -- For presence-only checks, use `true` instead of wildcards -- Patterns are case-sensitive - -### Pattern-Based Op Matching - -For complex op matching scenarios, use pattern objects with exclusions: - -```json -{ - "id": "llm_call", - "op": { - "pattern": "gen_ai.*", - "not": ["gen_ai.invoke_agent", "gen_ai.execute_tool"] - }, - "required_attributes": { ... } -} -``` - -This matches any span with op starting with `gen_ai.` EXCEPT `gen_ai.invoke_agent` and `gen_ai.execute_tool`. - -**Supported op formats:** - -- `"op": "gen_ai.chat"` - Single string (exact match) -- `"op": ["gen_ai.chat", "gen_ai.messages"]` - Array (OR matching) -- `"op": { "pattern": "gen_ai.*", "not": [...] }` - Pattern with exclusions - -### Schema Validation for Complex Attributes - -For attributes with structured data (like `gen_ai.request.messages`), use schema objects: - -```json -{ - "required_attributes": { - "gen_ai.request.messages": { - "type": "json_array", - "min_length": 2, - "items_have": ["role", "content"] - } - } -} -``` - -**Supported schema types:** - -**`json_array`** - Validates stringified JSON arrays or array objects - -- `length: N` - Exact array length -- `min_length: N` - Minimum array length -- `max_length: N` - Maximum array length -- `length_lte: "other.attribute.name"` - Array length must be less than or equal to another attribute's value -- `contains: "substring"` - Raw JSON string must contain this substring -- `items_have: ["prop1", "prop2"]` - All items must contain these properties - -**`plain_string`** - Validates plain strings (NOT stringified JSON) - -- `min_length: N` - Minimum string length -- `max_length: N` - Maximum string length -- `pattern: "value*"` - Wildcard pattern matching - -**`number`** - Validates numeric values with cross-attribute constraints - -- `lte: "other.attribute.name"` - Value must be less than or equal to another attribute -- `optional: true` - If attribute is missing, validation passes (applies to any schema type) - -**Optional Attribute Support:** - -All schema types support the `optional: true` property. When set: -- If the attribute exists → validate according to schema rules -- If the attribute is missing → validation passes (attribute is not required) - -This is useful for attributes that may or may not be present depending on the provider: - -```json -{ - "gen_ai.usage.input_tokens.cached": { - "type": "number", - "lte": "gen_ai.usage.input_tokens", - "optional": true - } -} -``` - -**Example use cases:** - -```json -{ - "gen_ai.request.messages": { - "type": "json_array", - "length": 2, - "items_have": ["role", "content"] - }, - "gen_ai.response.text": { - "type": "plain_string", - "min_length": 1, - "pattern": "*hello world*" - }, - "gen_ai.usage.input_tokens.cached": { - "type": "number", - "lte": "gen_ai.usage.input_tokens" - }, - "gen_ai.usage.output_tokens.reasoning": { - "type": "number", - "lte": "gen_ai.usage.output_tokens" - }, - "gen_ai.request.messages_truncated": { - "type": "json_array", - "min_length": 1, - "length_lte": "gen_ai.request.messages.original_length" - }, - "gen_ai.request.messages_with_redacted_binary": { - "type": "json_array", - "min_length": 1, - "contains": "[Blob substitute]" - } -} -``` - -### Shared Span Definitions - -To eliminate duplication, common span definitions are stored in `shared/specs/common-spans.json` and can be referenced using `$ref`: - -**common-spans.json:** -```json -{ - "llm_call": { - "id": "llm_call", - "op": { "pattern": "gen_ai.*", "not": [...] }, - "required_attributes": { ... } - }, - "invoke_agent": { ... } -} -``` - -**Using $ref in fixtures:** -```json -{ - "expectations": { - "spans": { - "items": [ - { "$ref": "common-spans#/llm_call" }, - { "$ref": "common-spans#/llm_call", "parent": "agent" } - ] - } - } -} -``` - -**Benefits:** -- Single source of truth for span definitions -- Properties in fixture override referenced span properties -- Reduces fixture size by ~50% - -### Order-Based Span Matching - -When multiple spans have the same op, they're matched in the order they appear in the fixture: - -```json -{ - "items": [ - { "$ref": "common-spans#/llm_call", "id": "first_call" }, - { "$ref": "common-spans#/llm_call", "id": "second_call" } - ] -} -``` - -First `llm_call` in fixture → first matching span -Second `llm_call` in fixture → second matching span (first excluded) - -**No occurrence field needed** - just list spans in expected order. - -## Writing New Specifications - -1. **Create directory:** `mkdir shared/specs/{spec-id}` -2. **Write spec.md:** Human-readable specification -3. **Create fixture-agentic.json:** Expectations for agentic frameworks -4. **Create fixture-low-level.json:** Expectations for low-level frameworks (if applicable) -5. **Implement test cases:** Add to SDKs in `sdks/js/*/cases/` and `sdks/py/*/cases/` - -## See Also - -- [Adding SDKs](../../sdks/README.md) - How to implement test cases -- [Test Utilities (JS)](../../sdks/js/_test-utils/README.md) - Fixture validation system -- [Test Utilities (Python)](../../sdks/py/_test-utils/README.md) - Fixture validation system -- [Main Documentation](../../CLAUDE.md) - Project overview diff --git a/shared/specs/common-spans.json b/shared/specs/common-spans.json deleted file mode 100644 index 815c40a..0000000 --- a/shared/specs/common-spans.json +++ /dev/null @@ -1,152 +0,0 @@ -{ - "llm_call": { - "id": "llm_call", - "op": { - "pattern": "gen_ai.*", - "not": [ - "gen_ai.invoke_agent", - "gen_ai.create_agent", - "gen_ai.execute_tool" - ] - }, - "required_attributes": { - "gen_ai.operation.name": true, - "gen_ai.request.model": "gpt-5-nano", - "gen_ai.response.model": "gpt-5-nano*", - "gen_ai.usage.input_tokens": true, - "gen_ai.usage.output_tokens": true, - "gen_ai.usage.total_tokens": true, - "gen_ai.usage.input_tokens.cached": { - "type": "number", - "lte": "gen_ai.usage.input_tokens", - "optional": true - }, - "gen_ai.usage.output_tokens.reasoning": { - "type": "number", - "lte": "gen_ai.usage.output_tokens", - "optional": true - }, - "gen_ai.response.text": true, - "gen_ai.request.messages": { - "type": "json_array", - "min_length": 2, - "items_have": ["role", "content"] - } - } - }, - "invoke_agent": { - "id": "invoke_agent", - "op": "gen_ai.invoke_agent", - "required_attributes": { - "gen_ai.operation.name": true, - "gen_ai.request.model": "gpt-5-nano", - "gen_ai.response.model": "gpt-5-nano*", - "gen_ai.usage.input_tokens": true, - "gen_ai.usage.output_tokens": true, - "gen_ai.usage.total_tokens": true, - "gen_ai.usage.input_tokens.cached": { - "type": "number", - "lte": "gen_ai.usage.input_tokens", - "optional": true - }, - "gen_ai.usage.output_tokens.reasoning": { - "type": "number", - "lte": "gen_ai.usage.output_tokens", - "optional": true - }, - "gen_ai.request.messages": { - "type": "json_array", - "min_length": 2, - "items_have": ["role", "content"] - } - } - }, - "agent_llm_call": { - "id": "llm_call", - "op": { - "pattern": "gen_ai.*", - "not": [ - "gen_ai.invoke_agent", - "gen_ai.create_agent", - "gen_ai.execute_tool" - ] - }, - "required_attributes": { - "gen_ai.agent.name": true, - "gen_ai.operation.name": true, - "gen_ai.request.model": "gpt-5-nano", - "gen_ai.response.model": "gpt-5-nano*", - "gen_ai.usage.input_tokens": true, - "gen_ai.usage.output_tokens": true, - "gen_ai.usage.total_tokens": true, - "gen_ai.response.text": true, - "gen_ai.request.messages": { - "type": "json_array", - "min_length": 2, - "items_have": ["role", "content"] - } - } - }, - "agent_invoke_agent": { - "id": "invoke_agent", - "op": "gen_ai.invoke_agent", - "required_attributes": { - "gen_ai.agent.name": true, - "gen_ai.operation.name": true, - "gen_ai.request.model": "gpt-5-nano", - "gen_ai.response.model": "gpt-5-nano*", - "gen_ai.usage.input_tokens": true, - "gen_ai.usage.output_tokens": true, - "gen_ai.usage.total_tokens": true, - "gen_ai.request.messages": { - "type": "json_array", - "min_length": 2, - "items_have": ["role", "content"] - } - } - }, - "agent_llm_call": { - "id": "llm_call", - "op": { - "pattern": "gen_ai.*", - "not": [ - "gen_ai.invoke_agent", - "gen_ai.create_agent", - "gen_ai.execute_tool" - ] - }, - "required_attributes": { - "gen_ai.agent.name": true, - "gen_ai.operation.name": true, - "gen_ai.request.model": "gpt-5-nano", - "gen_ai.response.model": "gpt-5-nano*", - "gen_ai.usage.input_tokens": true, - "gen_ai.usage.output_tokens": true, - "gen_ai.usage.total_tokens": true, - "gen_ai.response.text": true, - "gen_ai.request.messages": { - "type": "json_array", - "min_length": 2, - "items_have": ["role", "content"] - } - } - }, - "agent_invoke_agent": { - "id": "invoke_agent", - "op": "gen_ai.invoke_agent", - "required_attributes": { - "gen_ai.agent.name": true, - "gen_ai.operation.name": true, - "gen_ai.request.model": "gpt-5-nano", - "gen_ai.response.model": "gpt-5-nano*", - "gen_ai.usage.input_tokens": true, - "gen_ai.usage.output_tokens": true, - "gen_ai.usage.total_tokens": true, - "gen_ai.request.messages": { - "type": "json_array", - "min_length": 2, - "items_have": ["role", "content"] - } - } - } -} diff --git a/shared/specs/sdk-config-schema.json b/shared/specs/sdk-config-schema.json deleted file mode 100644 index d522764..0000000 --- a/shared/specs/sdk-config-schema.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "SDK Configuration Schema", - "description": "Configuration schema for AI SDK test parametrization. Allows per-SDK overrides of fixture expectations.", - "type": "object", - "required": ["sdk_name", "framework_type"], - "properties": { - "sdk_name": { - "type": "string", - "description": "Unique identifier for the SDK (e.g., 'google-genai', 'vercel', 'openai-agents')", - "examples": ["google-genai", "vercel", "openai-agents"] - }, - "framework_type": { - "type": "string", - "enum": ["agentic", "low-level"], - "description": "Framework type determines which fixture variant to use" - }, - "overrides": { - "type": "object", - "description": "Per-test-case overrides for fixture inputs and expectations", - "patternProperties": { - "^[0-9]+-[a-z-]+$": { - "type": "object", - "description": "Overrides for a specific test case (e.g., '1-simple', '2-simple-with-error')", - "properties": { - "model": { - "type": "string", - "description": "Model name to use in inputs.model (shorthand for common case)" - } - }, - "additionalProperties": { - "description": "Additional attribute overrides using dot notation paths (e.g., 'gen_ai.request.model', 'gen_ai.response.model')" - } - } - }, - "additionalProperties": false - }, - "metadata": { - "type": "object", - "description": "Optional metadata about the SDK", - "properties": { - "sdk_version": { - "type": "string", - "description": "SDK version being tested" - }, - "description": { - "type": "string", - "description": "Human-readable description of the SDK" - }, - "notes": { - "type": "string", - "description": "Additional notes or quirks about this SDK" - } - } - } - }, - "examples": [ - { - "sdk_name": "google-genai", - "framework_type": "low-level", - "overrides": { - "1-simple": { - "model": "gemini-2.5-flash-lite", - "gen_ai.request.model": "gemini-2.5-flash-lite", - "gen_ai.response.model": "gemini-2.5-flash-lite" - }, - "3-multi-turn": { - "model": "gemini-1.5-pro", - "gen_ai.request.model": "gemini-1.5-pro", - "gen_ai.response.model": "gemini-1.5-pro" - } - }, - "metadata": { - "sdk_version": "0.9.0", - "description": "Google Generative AI SDK", - "notes": "Uses Gemini models instead of OpenAI models" - } - } - ] -} diff --git a/shared/test-assets/README.md b/shared/test-assets/README.md deleted file mode 100644 index b4198e3..0000000 --- a/shared/test-assets/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Test Assets - -This directory contains static test assets used across test cases. - -## Files - -### test-image-10x10-red.png - -- **Size:** 10x10 pixels -- **Format:** PNG -- **Color:** Red (#FF0000) -- **Purpose:** Used in binary content redaction tests (test case 10) to verify that Sentry correctly redacts binary data in captured spans - -This static image replaces the need for dynamically generating test images with Pillow, reducing dependencies and improving test reliability. - -## Usage - -Test cases can read this image using standard file operations: - -**Python:** -```python -from pathlib import Path - -# Get path to test image (from SDK test case) -repo_root = Path(__file__).parent.parent.parent.parent -image_path = repo_root / "shared" / "test-assets" / "test-image-10x10-red.png" - -with open(image_path, "rb") as f: - image_data = f.read() -``` - -**JavaScript:** -```javascript -const fs = require('fs'); -const path = require('path'); - -// Get path to test image (from SDK test case) -const repoRoot = path.join(__dirname, '..', '..', '..', '..'); -const imagePath = path.join(repoRoot, 'shared', 'test-assets', 'test-image-10x10-red.png'); - -const imageData = fs.readFileSync(imagePath); -``` diff --git a/shared/test-assets/test-image-10x10-red.png b/shared/test-assets/test-image-10x10-red.png deleted file mode 100644 index 1ba6616..0000000 Binary files a/shared/test-assets/test-image-10x10-red.png and /dev/null differ diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 0000000..46f24f7 --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,283 @@ +#!/usr/bin/env node +/** + * CLI entry point + */ + +import "dotenv/config"; +import { parseArgs } from "node:util"; +import { Orchestrator } from "./orchestrator.js"; +import { FrameworkConfig } from "./types.js"; +import { + discoverFrameworks, + listFrameworks, +} from "./runner/framework-discovery.js"; +import { getAllTests } from "./test-cases/index.js"; + +const HELP_TEXT = ` +Sentry AI SDK Integration Tests + +Usage: + npm run test [command] [options] + +Commands: + run Run tests (default) + setup Setup environments and render templates (no test execution) + list List discovered frameworks + +Options: + --framework Filter by framework name + --test Filter by test name + --platform Filter by platform (js or py) + --sync Run only sync tests (default: both) + --async Run only async tests (default: both) + --streaming Run only streaming tests (default: both) + --blocking Run only blocking (non-streaming) tests (default: both) + --parallel, -j Run up to N tests in parallel (default: 1) + --verbose, -v Show detailed output (test execution logs, etc.) + --live-status Enable live status display (real-time tree view) + --open Open HTML report in browser after test run + --sentry-python Use local Sentry Python SDK (editable install) + --sentry-javascript Use local Sentry JavaScript SDK (link) + --help, -h Show this help message + +Examples: + npm run test list + npm run test run + npm run test -- --framework openai + npm run test -- --platform py --test "Basic LLM" + npm run test -- --platform py --sync + npm run test -- --platform py --async --verbose + npm run test -- --framework openai --live-status + npm run test -- --framework openai -j=4 + npm run test -- --framework openai --open + npm run test -- --framework openai --sentry-python ~/sentry-python + npm run test setup -- --framework openai --sync --streaming +`; + +function parseCliArgs() { + const { values, positionals } = parseArgs({ + args: process.argv.slice(2), + options: { + framework: { type: "string" }, + test: { type: "string" }, + platform: { type: "string" }, + sync: { type: "boolean", default: false }, + async: { type: "boolean", default: false }, + streaming: { type: "boolean", default: false }, + blocking: { type: "boolean", default: false }, + parallel: { type: "string", short: "j" }, + verbose: { type: "boolean", short: "v", default: false }, + "live-status": { type: "boolean", default: false }, + open: { type: "boolean", default: false }, + "sentry-python": { type: "string" }, + "sentry-javascript": { type: "string" }, + help: { type: "boolean", short: "h", default: false }, + }, + allowPositionals: true, + }); + + // Determine command from positionals + let command: "run" | "list" | "setup" = "run"; + if (positionals.includes("list")) { + command = "list"; + } else if (positionals.includes("setup")) { + command = "setup"; + } else if (positionals.includes("run")) { + command = "run"; + } + + // Parse parallel value + // Note: parseArgs returns "=2" for "-j=2" (short option with =), so we strip the leading = + let parallel: number | undefined; + if (values.parallel) { + const parallelStr = values.parallel.replace(/^=/, ""); + const parsed = parseInt(parallelStr, 10); + if (isNaN(parsed) || parsed < 1) { + console.error("Error: --parallel must be a positive integer"); + process.exit(1); + } + parallel = parsed; + } + + // Validate platform + const platform = values.platform as "js" | "py" | undefined; + if (platform && platform !== "js" && platform !== "py") { + console.error('Error: --platform must be "js" or "py"'); + process.exit(1); + } + + return { + command, + framework: values.framework, + test: values.test, + platform, + sync: values.sync, + async: values.async, + streaming: values.streaming, + blocking: values.blocking, + parallel, + verbose: values.verbose, + liveStatus: values["live-status"], + open: values.open, + sentryPythonPath: values["sentry-python"], + sentryJavaScriptPath: values["sentry-javascript"], + help: values.help, + }; +} + +async function main() { + const options = parseCliArgs(); + + if (options.help) { + console.log(HELP_TEXT); + process.exit(0); + } + + console.log("Sentry AI SDK Integration Tests\n"); + + // Handle list command + if (options.command === "list") { + listFrameworks(); + return; + } + + // Setup command doesn't need span collector or live status + const isSetupOnly = options.command === "setup"; + + const orchestrator = new Orchestrator({ + liveStatus: options.liveStatus, + verbose: options.verbose, + sync: options.sync, + async: options.async, + streaming: options.streaming, + blocking: options.blocking, + parallel: options.parallel, + openReport: options.open, + }); + + try { + // Start orchestrator (skip span collector for setup-only mode) + if (!isSetupOnly) { + await orchestrator.start(); + } + + // Discover frameworks + let discoveredFrameworks = discoverFrameworks(); + + // Apply filters + if (options.platform) { + discoveredFrameworks = discoveredFrameworks.filter( + (f) => f.platform === options.platform, + ); + } + if (options.framework) { + discoveredFrameworks = discoveredFrameworks.filter( + (f) => f.name === options.framework, + ); + } + + if (discoveredFrameworks.length === 0) { + console.log("No frameworks found matching criteria."); + await orchestrator.stop(); + return; + } + + // Load test definitions + let testDefinitions = getAllTests(); + if (options.test) { + testDefinitions = testDefinitions.filter((t) => t.name === options.test); + } + + if (testDefinitions.length === 0) { + console.log("No tests found matching criteria."); + await orchestrator.stop(); + return; + } + + // Set local Sentry SDK paths if provided + if (options.sentryPythonPath) { + process.env.SENTRY_PYTHON_PATH = options.sentryPythonPath; + console.log( + `Using local Sentry Python SDK: ${options.sentryPythonPath}\n`, + ); + } + if (options.sentryJavaScriptPath) { + process.env.SENTRY_JAVASCRIPT_PATH = options.sentryJavaScriptPath; + console.log( + `Using local Sentry JavaScript SDK: ${options.sentryJavaScriptPath}\n`, + ); + } + + // Convert discovered frameworks to test matrix + const frameworks: FrameworkConfig[] = discoveredFrameworks + .filter((df) => { + // Filter out frameworks with missing required arrays + if (!df.versions || df.versions.length === 0) { + console.warn( + `Warning: Framework '${df.name}' has no versions defined, skipping`, + ); + return false; + } + if (!df.sentryVersions || df.sentryVersions.length === 0) { + console.warn( + `Warning: Framework '${df.name}' has no sentryVersions defined, skipping`, + ); + return false; + } + return true; + }) + .map((df) => { + // Determine Sentry version based on platform and local SDK paths + let sentryVersion = df.sentryVersions[0]; + if (df.platform === "py" && options.sentryPythonPath) { + sentryVersion = "local"; + } else if (df.platform === "js" && options.sentryJavaScriptPath) { + sentryVersion = "local"; + } + + return { + name: df.name, + platform: df.platform, + type: df.type, + version: df.versions[0], + sentryVersion, + templatePath: df.templatePath, + category: df.category, + dependencies: df.dependencies, + executionMode: df.executionMode, + streamingMode: df.streamingMode, + modelOverrides: df.modelOverrides, + skip: df.skip, + }; + }); + + if (options.verbose) { + console.log( + `Testing ${frameworks.length} framework(s) with ${testDefinitions.length} test(s)\n`, + ); + } + + if (isSetupOnly) { + // Setup only - no test execution + await orchestrator.setupTests(frameworks, testDefinitions); + process.exit(0); + } else { + // Run tests + const report = await orchestrator.runTests(frameworks, testDefinitions); + + // Print report + orchestrator.printReport(report); + + // Exit with appropriate code + const exitCode = report.failed > 0 || report.errors > 0 ? 1 : 0; + await orchestrator.stop(); + process.exit(exitCode); + } + } catch (error) { + console.error("Fatal error:", error); + await orchestrator.stop(); + process.exit(1); + } +} + +main(); diff --git a/src/concurrency.ts b/src/concurrency.ts new file mode 100644 index 0000000..597f0e9 --- /dev/null +++ b/src/concurrency.ts @@ -0,0 +1,87 @@ +/** + * Simple concurrency limiter for parallel task execution. + * + * This provides a way to run async tasks with a maximum concurrency limit, + * similar to p-limit but without external dependencies. + */ + +/** + * Execution strategy interface for future extensibility. + * Allows implementing different strategies like: + * - Simple pool (current implementation) + * - Framework-grouped execution + * - Provider-aware rate limiting + */ +export interface ExecutionStrategy { + execute(tasks: T[], fn: (task: T) => Promise): Promise; +} + +/** + * Creates a concurrency limiter function. + * + * @param concurrency Maximum number of concurrent executions + * @returns A function that wraps async functions to limit concurrency + * + * @example + * const limit = createLimiter(3); + * const results = await Promise.all( + * tasks.map(task => limit(() => processTask(task))) + * ); + */ +export function createLimiter(concurrency: number) { + const queue: Array<() => void> = []; + let activeCount = 0; + + const next = () => { + activeCount--; + if (queue.length > 0) { + const nextFn = queue.shift()!; + nextFn(); + } + }; + + return (fn: () => Promise): Promise => { + return new Promise((resolve, reject) => { + const run = async () => { + activeCount++; + try { + const result = await fn(); + resolve(result); + } catch (error) { + reject(error); + } finally { + next(); + } + }; + + if (activeCount < concurrency) { + run(); + } else { + queue.push(run); + } + }); + }; +} + +/** + * Simple pool-based execution strategy. + * Runs up to N tasks concurrently, regardless of task type. + */ +export class PoolExecutionStrategy implements ExecutionStrategy { + constructor(private concurrency: number) {} + + async execute(tasks: T[], fn: (task: T) => Promise): Promise { + if (this.concurrency <= 1) { + // Sequential execution + const results: R[] = []; + for (const task of tasks) { + results.push(await fn(task)); + } + return results; + } + + // Parallel execution with concurrency limit + const limit = createLimiter(this.concurrency); + return Promise.all(tasks.map((task) => limit(() => fn(task)))); + } +} diff --git a/src/generate-html-report.ts b/src/generate-html-report.ts new file mode 100644 index 0000000..79bf49a --- /dev/null +++ b/src/generate-html-report.ts @@ -0,0 +1,80 @@ +#!/usr/bin/env node +/** + * CLI script to generate an HTML report from a CTRF JSON file + * + * Usage: + * npm run report + * npm run report test-results/ctrf-report-2024-01-15-120000.json + */ + +import { readFile } from "fs/promises"; +import { basename, dirname, join } from "path"; +import type { Report } from "ctrf"; +import { + generateHTML, + writeHTMLReport, + getTimestamp, +} from "./reporters/html-generator.js"; + +async function main() { + const args = process.argv.slice(2); + + if (args.length === 0 || args.includes("--help") || args.includes("-h")) { + console.log(` +Usage: npm run report [output-dir] + +Generate an HTML report from a CTRF JSON file. + +Arguments: + ctrf-json-file Path to the CTRF JSON report file + output-dir Optional: Directory to write the HTML report (default: same as input) + +Examples: + npm run report test-results/ctrf-report-2024-01-15-120000.json + npm run report test-results/ctrf-report.json ./reports +`); + process.exit(0); + } + + const inputFile = args[0]; + const outputDir = args[1] || dirname(inputFile); + + try { + // Read and parse CTRF JSON + console.log(`Reading CTRF report: ${inputFile}`); + const content = await readFile(inputFile, "utf-8"); + const report: Report = JSON.parse(content); + + // Validate it's a CTRF report + if (report.reportFormat !== "CTRF") { + console.error( + "Error: Input file does not appear to be a valid CTRF report", + ); + process.exit(1); + } + + // Generate HTML + console.log("Generating HTML report..."); + const htmlContent = generateHTML(report); + + // Extract timestamp from input filename or generate new one + const inputBasename = basename(inputFile, ".json"); + const timestampMatch = inputBasename.match(/(\d{4}-\d{2}-\d{2}-\d{6})$/); + const timestamp = timestampMatch ? timestampMatch[1] : getTimestamp(); + + // Write HTML report + const outputPath = await writeHTMLReport(htmlContent, outputDir, timestamp); + console.log(`✓ HTML report written to: ${outputPath}`); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + console.error(`Error: File not found: ${inputFile}`); + } else if (error instanceof SyntaxError) { + console.error(`Error: Invalid JSON in file: ${inputFile}`); + } else { + console.error("Error:", error); + } + process.exit(1); + } +} + +main(); diff --git a/src/orchestrator.ts b/src/orchestrator.ts new file mode 100644 index 0000000..0bc4413 --- /dev/null +++ b/src/orchestrator.ts @@ -0,0 +1,1079 @@ +/** + * Main orchestrator - coordinates test execution + */ + +import * as fs from "fs/promises"; +import * as path from "path"; +import { exec } from "child_process"; +import { SpanCollector } from "./span-collector/server.js"; +import { Runner } from "./runner/runner.js"; +import { Validator, ValidationError } from "./validator.js"; +import { + generateCTRFReport, + writeCTRFReport, + getTimestamp, +} from "./reporters/ctrf-reporter.js"; +import { generateHTML, writeHTMLReport } from "./reporters/html-generator.js"; +import { LiveStatusReporter } from "./reporters/live-status.js"; +import { PoolExecutionStrategy, ExecutionStrategy } from "./concurrency.js"; +import { + TestDefinition, + FrameworkConfig, + TestRun, + TestReport, + CapturedSpan, +} from "./types.js"; + +export class Orchestrator { + private spanCollector: SpanCollector; + private runner: Runner; + private validator: Validator; + private liveStatus: LiveStatusReporter; + private testRuns: TestRun[] = []; + private useLiveStatus: boolean = false; + private verbose: boolean = false; + private parallelism: number = 1; + private executionStrategy: ExecutionStrategy; + private openReport: boolean = false; + + private syncFilter?: boolean; + private asyncFilter?: boolean; + private streamingFilter?: boolean; + private blockingFilter?: boolean; + + constructor( + options: { + liveStatus?: boolean; + verbose?: boolean; + sync?: boolean; + async?: boolean; + streaming?: boolean; + blocking?: boolean; + parallel?: number; + openReport?: boolean; + } = {}, + ) { + this.spanCollector = new SpanCollector(); + this.runner = new Runner(); + this.validator = new Validator(); + this.liveStatus = new LiveStatusReporter(); + this.useLiveStatus = options.liveStatus === true; // Default to false (opt-in) + this.verbose = options.verbose === true; // Default to false + this.parallelism = options.parallel ?? 1; + this.executionStrategy = new PoolExecutionStrategy( + this.parallelism, + ); + this.syncFilter = options.sync; + this.asyncFilter = options.async; + this.streamingFilter = options.streaming; + this.blockingFilter = options.blocking; + this.openReport = options.openReport === true; + + // Set verbose on validator + this.validator.setVerbose(this.verbose); + } + + /** + * Start the orchestrator + */ + async start(): Promise { + await this.spanCollector.start(); + if (this.verbose) { + console.log( + `Span collector started on port ${this.spanCollector.getPort()}`, + ); + } + } + + /** + * Stop the orchestrator + */ + async stop(): Promise { + await this.spanCollector.stop(); + } + + /** + * Run tests for given frameworks and test definitions + */ + async runTests( + frameworks: FrameworkConfig[], + testDefinitions: TestDefinition[], + ): Promise { + const startTime = Date.now(); + + // Generate and filter test matrix + let testMatrix = this.generateTestMatrix(frameworks, testDefinitions); + testMatrix = this.applyFilters(testMatrix); + + // Print test tree + this.printTestTree(testMatrix); + + // Phase 1: Setup environments and render all templates first + console.log("Setting up environments and rendering templates...\n"); + const renderedTests = await this.setupAndRenderAll(testMatrix); + + // Print rendered files summary + this.printRenderedFiles(renderedTests); + + if (this.useLiveStatus) { + // Register all tests with live status + for (const testRun of testMatrix) { + this.liveStatus.registerTest(testRun); + } + + // Start live status display + this.liveStatus.start(); + } + + // Phase 2: Execute all tests (skip those that failed setup) + // Filter out tests that failed setup + const testsToRun = testMatrix.filter((testRun) => { + if (testRun.status === "error") { + if (this.verbose && !this.useLiveStatus) { + console.log( + `\n[${testRun.framework.name}] Skipping: ${testRun.testDefinition.name} (setup failed)`, + ); + } else if (!this.useLiveStatus) { + // Pytest-style progress: E for error + process.stdout.write("\x1b[33mE\x1b[0m"); + } + return false; + } + return true; + }); + + console.log( + `Executing ${testsToRun.length} test(s) with ${this.parallelism} worker(s)...\n`, + ); + + // Execute tests using the execution strategy (parallel or sequential) + await this.executionStrategy.execute(testsToRun, (testRun) => + this.executeTest(testRun), + ); + + // Stop live status display + if (this.useLiveStatus) { + this.liveStatus.stop(); + } + + // End progress line in non-verbose mode + if (!this.verbose && !this.useLiveStatus) { + console.log(""); // New line after progress dots + } + + // Generate report + const endTime = Date.now(); + const report = this.generateReport(startTime, endTime); + + // Generate and write reports (CTRF + HTML) + const htmlPath = await this.writeReports(report); + + // Open report in browser if requested + if (this.openReport && htmlPath) { + this.openInBrowser(htmlPath); + } + + return report; + } + + /** + * Setup environments and render templates for all tests + * Returns map of test run ID to rendered file path + * Also tracks which frameworks failed setup so their tests can be marked as errors + */ + private async setupAndRenderAll( + testMatrix: TestRun[], + ): Promise> { + const renderedTests = new Map(); + + // Group tests by framework to avoid redundant environment setup + const testsByFramework = new Map(); + for (const testRun of testMatrix) { + const key = `${testRun.framework.platform}/${testRun.framework.name}`; + if (!testsByFramework.has(key)) { + testsByFramework.set(key, []); + } + testsByFramework.get(key)!.push(testRun); + } + + // Setup each framework's environment once, then render all its templates + for (const [frameworkKey, runs] of testsByFramework) { + const firstRun = runs[0]; + const workDir = this.runner.getWorkDir(firstRun.framework); + + // Setup environment once per framework + if (this.verbose) { + console.log(`[${frameworkKey}] Setting up environment...`); + } + + const isAsync = + firstRun.framework.platform === "py" && + firstRun.framework.executionMode === "async"; + const isStreaming = firstRun.framework.streamingMode === "streaming"; + + try { + await this.runner.setupEnvironmentOnly({ + runId: firstRun.id, + framework: firstRun.framework, + testDefinition: firstRun.testDefinition, + sentryDsn: "https://dummy@sentry.io/123", // Dummy DSN for setup + workDir, + isAsync, + isStreaming, + verbose: this.verbose, + }); + } catch (setupError) { + // Mark all tests for this framework as errors + const errorMessage = + setupError instanceof Error ? setupError.message : String(setupError); + console.error(`[${frameworkKey}] Setup failed: ${errorMessage}`); + + for (const testRun of runs) { + testRun.status = "error"; + testRun.error = `Environment setup failed: ${errorMessage}`; + testRun.startTime = Date.now(); + testRun.endTime = Date.now(); + this.testRuns.push(testRun); + } + + // Skip to the next framework + continue; + } + + // Render all templates for this framework + for (const testRun of runs) { + const displayName = this.buildDisplayName(testRun); + if (this.verbose) { + console.log(`[${frameworkKey}] Rendering: ${displayName}`); + } + + const testIsAsync = + testRun.framework.platform === "py" && + testRun.framework.executionMode === "async"; + const testIsStreaming = testRun.framework.streamingMode === "streaming"; + + try { + const testPath = await this.runner.renderTemplateOnly({ + runId: testRun.id, + framework: testRun.framework, + testDefinition: testRun.testDefinition, + sentryDsn: "https://dummy@sentry.io/123", // Will be replaced during execution + workDir, + isAsync: testIsAsync, + isStreaming: testIsStreaming, + verbose: false, // Suppress template rendering logs, we're logging above + }); + + renderedTests.set(testRun.id, testPath); + } catch (renderError) { + // Mark this specific test as an error + const errorMessage = + renderError instanceof Error + ? renderError.message + : String(renderError); + console.error( + `[${frameworkKey}] Template rendering failed for ${displayName}: ${errorMessage}`, + ); + + testRun.status = "error"; + testRun.error = `Template rendering failed: ${errorMessage}`; + testRun.startTime = Date.now(); + testRun.endTime = Date.now(); + this.testRuns.push(testRun); + } + } + } + + return renderedTests; + } + + /** + * Print summary of rendered test files + */ + private printRenderedFiles(renderedTests: Map): void { + const colors = { + reset: "\x1b[0m", + dim: "\x1b[2m", + green: "\x1b[32m", + cyan: "\x1b[36m", + }; + + console.log( + `${colors.green}✓${colors.reset} Rendered ${renderedTests.size} test file(s)\n`, + ); + + if (this.verbose) { + // Group by directory for cleaner output + const byDir = new Map(); + for (const [_, filePath] of renderedTests) { + const dir = path.dirname(filePath); + const file = path.basename(filePath); + if (!byDir.has(dir)) { + byDir.set(dir, []); + } + byDir.get(dir)!.push(file); + } + + for (const [dir, files] of byDir) { + console.log(`${colors.dim}${dir}/${colors.reset}`); + for (const file of files) { + console.log(` ${colors.cyan}${file}${colors.reset}`); + } + } + console.log(""); + } + } + + /** + * Setup test environments and render templates without executing tests + */ + async setupTests( + frameworks: FrameworkConfig[], + testDefinitions: TestDefinition[], + ): Promise { + // Generate and filter test matrix (same as runTests) + let testMatrix = this.generateTestMatrix(frameworks, testDefinitions); + testMatrix = this.applyFilters(testMatrix); + + // Print test tree + this.printTestTree(testMatrix); + + console.log("Setting up test environments...\n"); + + // Setup each test (environment + template rendering only) + for (const testRun of testMatrix) { + await this.setupTest(testRun); + } + + console.log(`\n✓ Setup complete. ${testMatrix.length} test(s) prepared.`); + + // Print unique work directories + const uniqueWorkDirs = new Map(); + for (const testRun of testMatrix) { + const workDir = this.runner.getWorkDir(testRun.framework); + const key = `${testRun.framework.name}-${workDir}`; + if (!uniqueWorkDirs.has(key)) { + uniqueWorkDirs.set(key, workDir); + } + } + + console.log("\nWork directories:"); + for (const [_, workDir] of uniqueWorkDirs) { + console.log(` ${workDir}`); + } + } + + /** + * Setup a single test (environment + template) without executing + */ + private async setupTest(testRun: TestRun): Promise { + const displayName = this.buildDisplayName(testRun); + console.log(`[${testRun.framework.name}] Setting up: ${displayName}`); + + try { + // Determine isAsync and isStreaming flags + const isAsync = + testRun.framework.platform === "py" && + testRun.framework.executionMode === "async"; + const isStreaming = testRun.framework.streamingMode === "streaming"; + + // Setup environment and render template via runner (but don't execute) + await this.runner.setupOnly({ + runId: testRun.id, + framework: testRun.framework, + testDefinition: testRun.testDefinition, + sentryDsn: "https://dummy@sentry.io/123", // Dummy DSN for setup + workDir: this.runner.getWorkDir(testRun.framework), + isAsync, + isStreaming, + verbose: this.verbose, + }); + + console.log(` ✓ Setup complete`); + } catch (error) { + console.error( + ` ✗ Setup failed:`, + error instanceof Error ? error.message : error, + ); + } + } + + /** + * Apply sync/async and streaming/blocking filters to test matrix + */ + private applyFilters(testMatrix: TestRun[]): TestRun[] { + // Filter by sync/async if specified + if (this.syncFilter && !this.asyncFilter) { + testMatrix = testMatrix.filter((run) => { + if (run.framework.platform === "js") return false; + return run.framework.executionMode === "sync"; + }); + } else if (this.asyncFilter && !this.syncFilter) { + testMatrix = testMatrix.filter((run) => { + if (run.framework.platform === "js") return false; + return run.framework.executionMode === "async"; + }); + } + + // Filter by streaming/blocking if specified + if (this.streamingFilter && !this.blockingFilter) { + testMatrix = testMatrix.filter( + (run) => run.framework.streamingMode === "streaming", + ); + } else if (this.blockingFilter && !this.streamingFilter) { + testMatrix = testMatrix.filter( + (run) => run.framework.streamingMode === "blocking", + ); + } + + return testMatrix; + } + + /** + * Write CTRF and HTML reports to files + * Returns the path to the HTML report if successful + */ + async writeReports(report: TestReport): Promise { + const timestamp = getTimestamp(); + const outputDir = "./test-results"; + + // Write CTRF report + try { + const ctrfReport = generateCTRFReport(report); + const ctrfPath = await writeCTRFReport(ctrfReport, outputDir, timestamp); + if (this.verbose) { + console.log(`\n✓ CTRF report written to: ${ctrfPath}`); + } + + // Write HTML report + const htmlContent = generateHTML(ctrfReport); + const htmlPath = await writeHTMLReport(htmlContent, outputDir, timestamp); + if (this.verbose) { + console.log(`✓ HTML report written to: ${htmlPath}`); + } + + return htmlPath; + } catch (error) { + if (this.verbose) { + console.error("Failed to write reports:", error); + } + return undefined; + } + } + + /** + * Open a file in the default browser + */ + private openInBrowser(filePath: string): void { + const absolutePath = path.resolve(filePath); + const url = `file://${absolutePath}`; + + // Use platform-specific command to open browser + const platform = process.platform; + let command: string; + + if (platform === "darwin") { + command = `open "${url}"`; + } else if (platform === "win32") { + command = `start "" "${url}"`; + } else { + // Linux and others + command = `xdg-open "${url}"`; + } + + exec(command, (error) => { + if (error && this.verbose) { + console.error(`Failed to open browser: ${error.message}`); + } + }); + } + + /** + * Generate test matrix (framework × test definition combinations) + */ + private generateTestMatrix( + frameworks: FrameworkConfig[], + testDefinitions: TestDefinition[], + ): TestRun[] { + const matrix: TestRun[] = []; + + for (const framework of frameworks) { + for (const testDefinition of testDefinitions) { + // Check if test is explicitly skipped for this framework + if (framework.skip?.tests?.includes(testDefinition.name)) { + if (this.verbose) { + console.log( + `⊘ Skipping ${testDefinition.name} on ${framework.name} (explicitly skipped in config)`, + ); + } + continue; + } + + // Skip incompatible combinations based on test type + const isCompatible = this.isCompatible(framework, testDefinition); + + if (!isCompatible.compatible) { + if (this.verbose) { + console.log( + `⊘ Skipping ${testDefinition.name} on ${framework.name} (${isCompatible.reason})`, + ); + } + continue; + } + + // Generate test runs for all combinations of execution mode and streaming mode + const executionModes = this.getExecutionModes(framework); + const streamingModes = this.getStreamingModes(framework); + + for (const execMode of executionModes) { + for (const streamMode of streamingModes) { + const runId = this.generateRunId(); + matrix.push({ + id: runId, + index: matrix.length, // Track original order for consistent reporting + framework: { + ...framework, + executionMode: execMode, + streamingMode: streamMode, + }, + testDefinition, + status: "pending", + }); + } + } + } + } + + return matrix; + } + + /** + * Print a tree view of tests to be run + */ + private printTestTree(testMatrix: TestRun[]): void { + const colors = { + reset: "\x1b[0m", + bright: "\x1b[1m", + dim: "\x1b[2m", + cyan: "\x1b[36m", + gray: "\x1b[90m", + yellow: "\x1b[33m", + }; + + // Group by platform -> framework -> tests + const tree = new Map>(); + + for (const run of testMatrix) { + const platform = run.framework.platform; + const framework = run.framework.name; + + if (!tree.has(platform)) { + tree.set(platform, new Map()); + } + const platformMap = tree.get(platform)!; + + if (!platformMap.has(framework)) { + platformMap.set(framework, []); + } + platformMap.get(framework)!.push(run); + } + + console.log(`\n${colors.cyan}${colors.bright}Tests to run:${colors.reset}`); + + for (const [platform, frameworks] of tree) { + const platformIcon = platform === "py" ? "🐍" : "📦"; + console.log( + `${platformIcon} ${colors.bright}${platform.toUpperCase()}${colors.reset}`, + ); + + const frameworkEntries = Array.from(frameworks.entries()); + for (let fi = 0; fi < frameworkEntries.length; fi++) { + const [framework, runs] = frameworkEntries[fi]; + const isLastFramework = fi === frameworkEntries.length - 1; + const frameworkPrefix = isLastFramework ? "└─" : "├─"; + + console.log( + ` ${frameworkPrefix} ${colors.bright}${framework}${colors.reset}`, + ); + + for (let ri = 0; ri < runs.length; ri++) { + const run = runs[ri]; + const isLast = ri === runs.length - 1; + const testPrefix = isLastFramework ? " " : " │ "; + const testBranch = isLast ? "└─" : "├─"; + + // Build mode string with execution mode and streaming mode + const modeParts: string[] = []; + if (run.framework.executionMode) { + modeParts.push(run.framework.executionMode); + } + if (run.framework.streamingMode) { + modeParts.push(run.framework.streamingMode); + } + const mode = + modeParts.length > 0 + ? ` ${colors.dim}(${modeParts.join(", ")})${colors.reset}` + : ""; + + console.log( + `${testPrefix}${testBranch} ${run.testDefinition.name}${mode}`, + ); + } + } + } + + console.log( + `\n${colors.dim}Total: ${testMatrix.length} test(s)${colors.reset}\n`, + ); + } + + /** + * Check if a test is compatible with a framework + */ + private isCompatible( + framework: FrameworkConfig, + test: TestDefinition, + ): { compatible: boolean; reason?: string } { + // LLM tests can only run on llm-only frameworks + if (test.type === "llm" && framework.type !== "llm-only") { + return { + compatible: false, + reason: "LLM test requires llm-only framework", + }; + } + + // Agent tests can only run on agentic frameworks + if (test.type === "agent" && framework.type !== "agentic") { + return { + compatible: false, + reason: "Agent test requires agentic framework", + }; + } + + return { compatible: true }; + } + + /** + * Get execution modes to test for a framework + */ + private getExecutionModes( + framework: FrameworkConfig, + ): Array<"sync" | "async" | undefined> { + // JavaScript doesn't have sync/async distinction at the framework level + if (framework.platform === "js") { + return [undefined]; + } + + // Python: expand "both" to sync and async + if (framework.executionMode === "both") { + return ["sync", "async"]; + } + + // Return single mode or undefined + return [framework.executionMode]; + } + + /** + * Get streaming modes to test for a framework + */ + private getStreamingModes( + framework: FrameworkConfig, + ): Array<"streaming" | "blocking" | undefined> { + // If streaming mode is "both", expand to both variants + if (framework.streamingMode === "both") { + return ["streaming", "blocking"]; + } + + // Return single mode or undefined (for frameworks that don't specify streaming) + return [framework.streamingMode]; + } + + /** + * Execute a single test + */ + private async executeTest(testRun: TestRun): Promise { + this.testRuns.push(testRun); + testRun.status = "running"; + testRun.startTime = Date.now(); + + // Update live status + if (this.useLiveStatus) { + this.liveStatus.updateTestStatus(testRun, "running"); + } + + // Build display name with mode suffixes + const displayName = this.buildDisplayName(testRun); + + if (this.verbose && !this.useLiveStatus) { + console.log(`\n[${testRun.framework.name}] Running: ${displayName}`); + } + + try { + // Register run with span collector + this.spanCollector.registerRun(testRun.id); + + // Get DSN for this test run + const sentryDsn = this.spanCollector.getDsn(testRun.id); + + // Determine isAsync flag for Python frameworks + const isAsync = + testRun.framework.platform === "py" && + testRun.framework.executionMode === "async"; + + // Determine isStreaming flag + const isStreaming = testRun.framework.streamingMode === "streaming"; + + // Execute test via runner (template already rendered in setup phase) + await this.runner.executeOnly({ + runId: testRun.id, + framework: testRun.framework, + testDefinition: testRun.testDefinition, + sentryDsn, + workDir: this.runner.getWorkDir(testRun.framework), + isAsync, + isStreaming, + verbose: this.verbose && !this.useLiveStatus, // Only verbose when flag is set and not live status + }); + + // Wait for spans to be collected + await this.waitForSpans(testRun.id); + + // Get captured spans + const spans = this.spanCollector.getSpans(testRun.id); + testRun.spans = spans; + + // Append spans to log file (always, with full detail) + await this.appendSpansToLogFile(testRun, spans); + + // Validate spans using test definition's check methods + const checkResults = await this.validator.validate( + spans, + testRun.testDefinition, + testRun.framework, + // Pass callback to update live status for each check + this.useLiveStatus + ? (checkName: string) => { + this.liveStatus.updateCurrentCheck(testRun, checkName); + } + : undefined, + // Pass callback to update check result + this.useLiveStatus + ? (checkResult) => { + this.liveStatus.updateCheckResult(testRun, checkResult); + } + : undefined, + ); + testRun.checkResults = checkResults; + + testRun.status = "passed"; + + // Update live status + if (this.useLiveStatus) { + this.liveStatus.updateTestStatus(testRun, "passed"); + } + + if (this.verbose && !this.useLiveStatus) { + console.log(`✓ ${displayName} passed`); + } else if (!this.useLiveStatus) { + // Pytest-style progress: dot for passed + process.stdout.write("\x1b[32m.\x1b[0m"); + } + } catch (error) { + testRun.status = "failed"; + + // Extract check results from ValidationError if available + if (error instanceof ValidationError) { + testRun.checkResults = error.checkResults; + testRun.error = error.message; + } else { + testRun.error = error instanceof Error ? error.message : String(error); + } + + // Update live status + if (this.useLiveStatus) { + this.liveStatus.updateTestStatus(testRun, "failed", testRun.error); + } + + if (this.verbose && !this.useLiveStatus) { + console.error(`✗ ${displayName} failed:`, testRun.error); + } else if (!this.useLiveStatus) { + // Pytest-style progress: F for failed + process.stdout.write("\x1b[31mF\x1b[0m"); + } + } finally { + testRun.endTime = Date.now(); + this.spanCollector.clearRun(testRun.id); + } + } + + /** + * Wait for spans to be collected (with timeout) + */ + private async waitForSpans( + runId: string, + timeoutMs: number = 5000, + ): Promise { + const startTime = Date.now(); + const checkInterval = 100; + + while (Date.now() - startTime < timeoutMs) { + const spans = this.spanCollector.getSpans(runId); + if (spans.length > 0) { + // Give a bit more time for additional spans + await new Promise((resolve) => setTimeout(resolve, 500)); + return; + } + await new Promise((resolve) => setTimeout(resolve, checkInterval)); + } + + // No spans received - this might be expected for error tests + console.warn(`No spans received for run ${runId} after ${timeoutMs}ms`); + } + + /** + * Generate report + */ + private generateReport(startTime: number, endTime: number): TestReport { + // Sort runs by original index for consistent ordering (parallel execution + // may complete tests in arbitrary order) + const sortedRuns = [...this.testRuns].sort( + (a, b) => (a.index ?? 0) - (b.index ?? 0), + ); + + const passed = sortedRuns.filter((r) => r.status === "passed").length; + const failed = sortedRuns.filter((r) => r.status === "failed").length; + const errors = sortedRuns.filter((r) => r.status === "error").length; + const skipped = sortedRuns.filter((r) => r.status === "skipped").length; + + return { + totalTests: sortedRuns.length, + passed, + failed, + errors, + skipped, + duration: endTime - startTime, + runs: sortedRuns, + }; + } + + /** + * Generate unique run ID + */ + private generateRunId(): string { + return `run-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`; + } + + /** + * Build display name with mode suffixes (e.g., "Basic LLM Test (sync, streaming)") + */ + private buildDisplayName(testRun: TestRun): string { + const modeParts: string[] = []; + + // Add execution mode for Python + if ( + testRun.framework.platform === "py" && + testRun.framework.executionMode + ) { + modeParts.push(testRun.framework.executionMode); + } + + // Add streaming mode if specified + if (testRun.framework.streamingMode) { + modeParts.push(testRun.framework.streamingMode); + } + + if (modeParts.length > 0) { + return `${testRun.testDefinition.name} (${modeParts.join(", ")})`; + } + + return testRun.testDefinition.name; + } + + /** + * Append captured spans to the test log file + */ + private async appendSpansToLogFile( + testRun: TestRun, + spans: CapturedSpan[], + ): Promise { + try { + const workDir = this.runner.getWorkDir(testRun.framework); + const testCaseId = testRun.testDefinition.name + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); + + const modeParts: string[] = []; + if (testRun.framework.executionMode) { + modeParts.push(testRun.framework.executionMode); + } + if (testRun.framework.streamingMode) { + modeParts.push(testRun.framework.streamingMode); + } + const modeSuffix = modeParts.length > 0 ? modeParts.join("-") : "default"; + const logFile = path.join( + workDir, + `test-${testCaseId}-${modeSuffix}.log`, + ); + + // Build spans content with full JSON detail + const lines: string[] = [ + "", + "=== CAPTURED SPANS ===", + `Total spans: ${spans.length}`, + "", + ]; + + if (spans.length === 0) { + lines.push("(no spans captured)"); + } else { + for (let i = 0; i < spans.length; i++) { + const span = spans[i]; + lines.push(`--- Span ${i + 1} ---`); + lines.push(JSON.stringify(span, null, 2)); + lines.push(""); + } + } + + // Append to log file + await fs.appendFile(logFile, lines.join("\n")); + } catch (error) { + // Silently ignore errors writing to log file + console.error(" Warning: Could not append spans to log file:", error); + } + } + + /** + * Print test report with colors and detailed check breakdown + */ + printReport(report: TestReport): void { + // ANSI color codes + const colors = { + reset: "\x1b[0m", + bright: "\x1b[1m", + dim: "\x1b[2m", + green: "\x1b[32m", + red: "\x1b[31m", + yellow: "\x1b[33m", + blue: "\x1b[34m", + cyan: "\x1b[36m", + gray: "\x1b[90m", + }; + + console.log("\n" + colors.bright + "=".repeat(70) + colors.reset); + console.log( + colors.bright + colors.cyan + "📊 Test Results Summary" + colors.reset, + ); + console.log(colors.bright + "=".repeat(70) + colors.reset); + + // Summary stats with colors + console.log( + `${colors.bright}Total Tests:${colors.reset} ${report.totalTests}`, + ); + console.log(`${colors.green}✓ Passed:${colors.reset} ${report.passed}`); + console.log(`${colors.red}✗ Failed:${colors.reset} ${report.failed}`); + if (report.skipped > 0) { + console.log( + `${colors.yellow}⊘ Skipped:${colors.reset} ${report.skipped}`, + ); + } + if (report.errors > 0) { + console.log( + `${colors.yellow}⚠ Errors:${colors.reset} ${report.errors}`, + ); + } + console.log( + `${colors.blue}⏱ Duration:${colors.reset} ${(report.duration / 1000).toFixed(2)}s`, + ); + console.log(colors.bright + "=".repeat(70) + colors.reset); + + // Detailed test breakdown + console.log( + "\n" + colors.bright + colors.cyan + "📋 Detailed Results" + colors.reset, + ); + console.log(colors.gray + "─".repeat(70) + colors.reset); + + for (const run of report.runs) { + // Build mode string with execution mode (Python) and streaming mode + const modeParts: string[] = []; + if (run.framework.platform === "py" && run.framework.executionMode) { + modeParts.push(run.framework.executionMode); + } + if (run.framework.streamingMode) { + modeParts.push(run.framework.streamingMode); + } + const modeStr = + modeParts.length > 0 + ? ` ${colors.dim}(${modeParts.join(", ")})${colors.reset}` + : ""; + + // Test header + if (run.status === "passed") { + console.log( + `\n${colors.green}✓${colors.reset} ${colors.bright}[${run.framework.name}]${colors.reset} ${run.testDefinition.name}${modeStr}`, + ); + } else { + console.log( + `\n${colors.red}✗${colors.reset} ${colors.bright}[${run.framework.name}]${colors.reset} ${run.testDefinition.name}${modeStr}`, + ); + } + + // Check results breakdown + if (run.checkResults && run.checkResults.length > 0) { + for (const check of run.checkResults) { + if (check.status === "passed") { + console.log( + ` ${colors.green}✓${colors.reset} ${colors.dim}${check.name}${colors.reset}`, + ); + } else if (check.status === "skipped") { + const reason = check.skipReason || "Not supported"; + console.log( + ` ${colors.yellow}⊘${colors.reset} ${colors.dim}${check.name}${colors.reset} ${colors.gray}(${reason})${colors.reset}`, + ); + } else { + console.log( + ` ${colors.red}✗${colors.reset} ${colors.bright}${check.name}${colors.reset}`, + ); + if (check.error) { + // Print error message with indentation + const errorLines = check.error.split("\n"); + for (const line of errorLines) { + console.log(` ${colors.dim}${line}${colors.reset}`); + } + } + } + } + } else if (run.status === "skipped") { + // Show skip reason for skipped tests + const reason = run.skipReason || "Test skipped"; + console.log( + ` ${colors.yellow}⊘${colors.reset} ${colors.dim}${reason}${colors.reset}`, + ); + } else if (run.status === "failed" && run.error) { + // Fallback for tests without check results + console.log( + ` ${colors.red}Error:${colors.reset} ${colors.dim}${run.error}${colors.reset}`, + ); + } + + // Duration + if (run.startTime && run.endTime) { + const duration = ((run.endTime - run.startTime) / 1000).toFixed(2); + console.log(` ${colors.gray}⏱ ${duration}s${colors.reset}`); + } + } + + console.log("\n" + colors.gray + "─".repeat(70) + colors.reset); + + // Final summary + if (report.failed === 0 && report.errors === 0) { + console.log( + `\n${colors.green}${colors.bright}✓ All tests passed!${colors.reset} 🎉\n`, + ); + } else { + console.log( + `\n${colors.red}${colors.bright}✗ ${report.failed + report.errors} test(s) failed${colors.reset}\n`, + ); + } + } +} diff --git a/src/reporters/ctrf-reporter.ts b/src/reporters/ctrf-reporter.ts new file mode 100644 index 0000000..ef1531f --- /dev/null +++ b/src/reporters/ctrf-reporter.ts @@ -0,0 +1,168 @@ +/** + * CTRF Reporter - Converts TestReport to CTRF (Common Test Report Format) + * + * CTRF Specification: https://ctrf.io/ + */ + +import type { Report, Test } from "ctrf"; +import { TestReport, TestRun } from "../types.js"; +import * as fs from "fs/promises"; +import * as path from "path"; + +/** + * Convert TestReport to CTRF format + */ +export function generateCTRFReport(testReport: TestReport): Report { + const now = Date.now(); + const startTime = now - testReport.duration; + + // Convert each TestRun to CTRF Test + const tests: Test[] = testReport.runs.map((run) => { + const frameworkName = `${run.framework.platform}/${run.framework.name}`; + // Build mode string with execution mode (Python) and streaming mode + const modeParts: string[] = []; + if (run.framework.platform === "py" && run.framework.executionMode) { + modeParts.push(run.framework.executionMode); + } + if (run.framework.streamingMode) { + modeParts.push(run.framework.streamingMode); + } + const modeStr = modeParts.length > 0 ? ` (${modeParts.join(", ")})` : ""; + const testName = `${frameworkName} :: ${run.testDefinition.name}${modeStr}`; + + const test: Test = { + name: testName, + status: mapStatus(run.status), + duration: run.endTime && run.startTime ? run.endTime - run.startTime : 0, + }; + + // Add suite (grouping) + test.suite = [frameworkName]; + + // Add tags for filtering + const tags: string[] = [ + run.framework.platform, // 'py' or 'js' + run.framework.type, // 'llm-only' or 'agentic' + run.testDefinition.type, // 'llm' or 'agent' + ]; + + if (run.framework.executionMode) { + tags.push(run.framework.executionMode); // 'sync' or 'async' + } + if (run.framework.streamingMode) { + tags.push(run.framework.streamingMode); // 'streaming' or 'blocking' + } + + test.tags = tags; + + // Add error details if test failed + if (run.error) { + test.message = run.error.split("\n")[0]; // First line + test.trace = run.error; + } + + // Add extra metadata + test.extra = { + framework: run.framework.name, + frameworkVersion: run.framework.version, + sentryVersion: run.framework.sentryVersion, + testType: run.testDefinition.type, + platform: run.framework.platform, + ...(run.framework.executionMode && { + executionMode: run.framework.executionMode, + }), + ...(run.framework.streamingMode && { + streamingMode: run.framework.streamingMode, + }), + ...(run.spans && { + spanCount: run.spans.length, + spans: run.spans, + }), + }; + + return test; + }); + + // Calculate summary + const summary = { + tests: testReport.totalTests, + passed: testReport.passed, + failed: testReport.failed, + pending: 0, + skipped: testReport.skipped, + other: testReport.errors, + start: startTime, + stop: now, + }; + + // Build CTRF report + const report: Report = { + reportFormat: "CTRF", + specVersion: "1.0.0", + results: { + tool: { + name: "sentry-ai-sdk-test", + version: "1.0.0", + }, + summary, + tests, + }, + }; + + return report; +} + +/** + * Map our status to CTRF status + */ +function mapStatus( + status: string, +): "passed" | "failed" | "skipped" | "pending" | "other" { + switch (status) { + case "passed": + return "passed"; + case "failed": + return "failed"; + case "skipped": + return "skipped"; + case "error": + return "other"; + case "pending": + return "pending"; + default: + return "other"; + } +} + +/** + * Generate timestamp string for filenames + * Format: YYYY-MM-DD-HHmmss + */ +export function getTimestamp(): string { + const now = new Date(); + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, "0"); + const day = String(now.getDate()).padStart(2, "0"); + const hours = String(now.getHours()).padStart(2, "0"); + const minutes = String(now.getMinutes()).padStart(2, "0"); + const seconds = String(now.getSeconds()).padStart(2, "0"); + return `${year}-${month}-${day}-${hours}${minutes}${seconds}`; +} + +/** + * Write CTRF report to file + */ +export async function writeCTRFReport( + report: Report, + outputDir: string = "./test-results", + timestamp?: string, +): Promise { + // Ensure output directory exists + await fs.mkdir(outputDir, { recursive: true }); + + const ts = timestamp || getTimestamp(); + const filePath = path.join(outputDir, `ctrf-report-${ts}.json`); + await fs.writeFile(filePath, JSON.stringify(report, null, 2), "utf-8"); + + return filePath; +} diff --git a/src/reporters/html-generator.ts b/src/reporters/html-generator.ts new file mode 100644 index 0000000..6a431d1 --- /dev/null +++ b/src/reporters/html-generator.ts @@ -0,0 +1,682 @@ +/** + * HTML Generator - Reads CTRF Report and generates HTML report + * + * Uses htm+vhtml for templating (no build step required) + */ + +import htm from "htm"; +import vhtml from "vhtml"; +import type { Report, Test } from "ctrf"; +import { writeFile, mkdir } from "fs/promises"; +import { join } from "path"; + +const html = htm.bind(vhtml); + +/** + * Format duration in milliseconds to human-readable format + */ +function formatDuration(ms: number): string { + if (ms < 1000) return `${ms}ms`; + return `${(ms / 1000).toFixed(2)}s`; +} + +/** + * Get status icon for test result + */ +function getStatusIcon(status: string): string { + switch (status) { + case "passed": + return "✓"; + case "failed": + return "✗"; + case "skipped": + return "○"; + default: + return "-"; + } +} + +/** + * Generate summary cards HTML + */ +function SummaryCards({ summary }: { summary: Report["results"]["summary"] }) { + const duration = summary.stop - summary.start; + + return html` +
+
+

Total Tests

+

${summary.tests}

+
+
+

✓ Passed

+

${summary.passed}

+
+
+

✗ Failed

+

${summary.failed}

+
+
+

Duration

+

${formatDuration(duration)}

+
+
+ `; +} + +/** + * Natural sort comparator that handles numeric prefixes correctly. + * E.g., "1-simple" < "2-multi" < "10-binary" (not lexical "1" < "10" < "2") + */ +function naturalSortCompare(a: string, b: string): number { + // Extract numeric prefix if present (e.g., "10-binary" -> 10) + const aMatch = a.match(/^(\d+)/); + const bMatch = b.match(/^(\d+)/); + + // If both have numeric prefixes, compare numerically + if (aMatch && bMatch) { + const aNum = parseInt(aMatch[1], 10); + const bNum = parseInt(bMatch[1], 10); + if (aNum !== bNum) { + return aNum - bNum; + } + // If numeric prefixes are equal, compare the rest lexically + return a.localeCompare(b); + } + + // If only one has a numeric prefix, it comes first + if (aMatch) return -1; + if (bMatch) return 1; + + // Neither has a numeric prefix, compare lexically + return a.localeCompare(b); +} + +/** + * Extract base test name without mode suffixes + * e.g., "Basic LLM Test (async, streaming)" -> "Basic LLM Test" + */ +function getBaseTestName(testName: string): string { + // Remove the mode suffix in parentheses + return testName.replace(/\s*\([^)]*\)\s*$/, "").trim(); +} + +/** + * Combined status for multiple test variations + */ +interface CombinedTestResult { + passed: number; + failed: number; + skipped: number; + other: number; + total: number; + variations: Array<{ + mode: string; + status: string; + }>; +} + +/** + * Get overall status from combined results + */ +function getCombinedStatus(result: CombinedTestResult): string { + if (result.failed > 0) return "failed"; + if (result.other > 0) return "failed"; // errors count as failed + if (result.passed > 0 && result.skipped === 0) return "passed"; + if (result.passed > 0) return "partial"; // some passed, some skipped + if (result.skipped > 0) return "skipped"; + return "not-run"; +} + +/** + * Generate cell content with status icons for each variation + */ +function CombinedStatusCell({ result }: { result: CombinedTestResult }) { + const overallStatus = getCombinedStatus(result); + + // If only one variation, show simple icon with tooltip + if (result.total === 1) { + const v = result.variations[0]; + return html`${getStatusIcon(v.status)}`; + } + + // Multiple variations - show mini icons with tooltips + return html` + +
+ ${result.variations.map( + (v) => html` + + ${getStatusIcon(v.status)} + + `, + )} +
+ + `; +} + +/** + * Build test matrix for a specific test type (LLM or Agent) + */ +function TestMatrixByType({ + tests, + testType, + title, +}: { + tests: Test[]; + testType: string; + title: string; +}) { + // Filter tests by type + const filteredTests = tests.filter( + (t) => (t.extra as Record)?.testType === testType, + ); + + if (filteredTests.length === 0) { + return html``; + } + + // Extract unique SDKs + const sdks = [ + ...new Set( + filteredTests.map((t: Test) => + t.suite && t.suite.length > 0 ? t.suite[0] : "unknown", + ), + ), + ].sort(); + + // Extract unique base test names (without mode suffixes) + const testCases = [ + ...new Set( + filteredTests.map((t: Test) => { + const fullName = t.name.split(" :: ")[1] || t.name; + return getBaseTestName(fullName); + }), + ), + ].sort(naturalSortCompare); + + // Build lookup map: sdk::baseTestName -> CombinedTestResult + const testMap = new Map(); + + for (const test of filteredTests) { + const fullName = test.name.split(" :: ")[1] || test.name; + const baseName = getBaseTestName(fullName); + const suite = + test.suite && test.suite.length > 0 ? test.suite[0] : "unknown"; + const key = `${suite}::${baseName}`; + + // Extract mode from the test name (e.g., "(async, streaming)") + const modeMatch = fullName.match(/\(([^)]+)\)$/); + const mode = modeMatch ? modeMatch[1] : "default"; + + if (!testMap.has(key)) { + testMap.set(key, { + passed: 0, + failed: 0, + skipped: 0, + other: 0, + total: 0, + variations: [], + }); + } + + const result = testMap.get(key)!; + result.total++; + result.variations.push({ mode, status: test.status }); + + switch (test.status) { + case "passed": + result.passed++; + break; + case "failed": + result.failed++; + break; + case "skipped": + result.skipped++; + break; + default: + result.other++; + } + } + + return html` +

${title}

+ + + + + ${testCases.map((caseId) => html``)} + + + + ${sdks.map( + (sdk) => html` + + + ${testCases.map((caseId) => { + const key = `${sdk}::${caseId}`; + const result = testMap.get(key); + + if (!result) { + return html``; + } + + return CombinedStatusCell({ result }); + })} + + `, + )} + +
SDK${caseId}
${sdk}-
+ `; +} + +/** + * Build test matrices split by type (LLM and Agent) + */ +function TestMatrix({ report }: { report: Report }) { + return html` + ${TestMatrixByType({ + tests: report.results.tests, + testType: "llm", + title: "LLM Tests", + })} + ${TestMatrixByType({ + tests: report.results.tests, + testType: "agent", + title: "Agent Tests", + })} + `; +} + +/** + * Render spans as JSON for display + */ +function formatSpans(spans: unknown[]): string { + return JSON.stringify(spans, null, 2); +} + +/** + * Failed tests details section + */ +function FailedTestsDetails({ tests }: { tests: Test[] }) { + const failedTests = tests.filter((t) => t.status === "failed"); + + if (failedTests.length === 0) { + return html``; + } + + return html` +

Failed Tests Details

+ ${failedTests.map((test) => { + const caseId = test.name.split(" :: ")[1] || test.name; + const extra = test.extra as Record | undefined; + const spans = extra?.spans as unknown[] | undefined; + const spanCount = extra?.spanCount as number | undefined; + + return html` +
+ + ✗ + ${test.suite && test.suite.length > 0 + ? test.suite[0] + : "unknown"} + :: ${caseId} + (${test.duration}ms) + +
+ ${test.trace + ? html` +
+ Details: + 
${test.trace}
+
+ ` + : ""} + ${spans && spans.length > 0 + ? html` +
+ + {} + Captured Spans (${spanCount || spans.length}) + + 
${formatSpans(spans)}
+
+ ` + : spanCount === 0 + ? html`
No spans captured
` + : ""} +
+
+ `; + })} + `; +} + +/** + * Generate complete HTML report from CTRF report + */ +export function generateHTML(report: Report): string { + const title = "Sentry AI SDK Test Report"; + const timestamp = new Date(report.results.summary.stop).toLocaleString(); + + const htmlContent = html` + + + + + + ${title} + + + +
+

${title}

+

Generated: ${timestamp}

+ ${SummaryCards({ summary: report.results.summary })} + ${TestMatrix({ report })} + ${FailedTestsDetails({ tests: report.results.tests })} +
+ + + `; + + // vhtml returns mixed content: strings for HTML tags, and arrays for special elements like DOCTYPE + // The structure is typically: ["!DOCTYPE", attrs, "..."] + function flattenToString(value: unknown): string { + if (typeof value === "string") { + return value; + } + if (Array.isArray(value)) { + // Check if this is a DOCTYPE declaration + if (value[0] === "!DOCTYPE") { + // DOCTYPE + rest of HTML + return ( + "\n" + value.slice(2).map(flattenToString).join("") + ); + } + // Regular array, flatten all elements + return value.map(flattenToString).join(""); + } + if (typeof value === "object" && value !== null) { + // Skip objects (like attributes) + return ""; + } + return String(value); + } + + return flattenToString(htmlContent); +} + +/** + * Generate timestamp string for filenames + * Format: YYYY-MM-DD-HHmmss + */ +export function getTimestamp(): string { + const now = new Date(); + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, "0"); + const day = String(now.getDate()).padStart(2, "0"); + const hours = String(now.getHours()).padStart(2, "0"); + const minutes = String(now.getMinutes()).padStart(2, "0"); + const seconds = String(now.getSeconds()).padStart(2, "0"); + return `${year}-${month}-${day}-${hours}${minutes}${seconds}`; +} + +/** + * Write HTML report to file + */ +export async function writeHTMLReport( + htmlContent: string, + outputDir: string = "./test-results", + timestamp?: string, +): Promise { + // Ensure output directory exists + await mkdir(outputDir, { recursive: true }); + + const ts = timestamp || getTimestamp(); + const filePath = join(outputDir, `test-report-${ts}.html`); + await writeFile(filePath, htmlContent, "utf-8"); + + return filePath; +} diff --git a/src/reporters/live-status.ts b/src/reporters/live-status.ts new file mode 100644 index 0000000..62f7c61 --- /dev/null +++ b/src/reporters/live-status.ts @@ -0,0 +1,276 @@ +/** + * Live Status Reporter - Real-time terminal UI for test execution + * + * Displays a tree view of running tests with auto-updating status + * Uses log-update for flicker-free terminal updates + */ + +import logUpdate from 'log-update'; +import { TestRun, CheckResult } from '../types.js'; + +interface TestState { + framework: string; + platform: string; + type: string; + testName: string; + executionMode?: string; + status: 'pending' | 'running' | 'passed' | 'failed' | 'skipped'; + currentCheck?: string; + checkResults: CheckResult[]; + error?: string; + startTime?: number; +} + +export class LiveStatusReporter { + private states: Map = new Map(); + private isActive = false; + private renderInterval?: NodeJS.Timeout; + + /** + * Start the live status display + */ + start(): void { + if (this.isActive) return; + + this.isActive = true; + + // Render every 100ms + this.renderInterval = setInterval(() => { + this.render(); + }, 100); + + // Initial render + this.render(); + } + + /** + * Stop the live status display + */ + stop(): void { + if (!this.isActive) return; + + this.isActive = false; + + if (this.renderInterval) { + clearInterval(this.renderInterval); + this.renderInterval = undefined; + } + + // Stop log-update and persist the final output + logUpdate.done(); + } + + /** + * Register a test run + */ + registerTest(testRun: TestRun): void { + const key = this.getKey(testRun); + this.states.set(key, { + framework: testRun.framework.name, + platform: testRun.framework.platform, + type: testRun.framework.type, + testName: testRun.testDefinition.name, + executionMode: testRun.framework.executionMode, + status: 'pending', + checkResults: [], + }); + } + + /** + * Update test status + */ + updateTestStatus(testRun: TestRun, status: 'running' | 'passed' | 'failed' | 'skipped', error?: string): void { + const key = this.getKey(testRun); + const state = this.states.get(key); + if (state) { + state.status = status; + state.error = error; + if (status === 'running' && !state.startTime) { + state.startTime = Date.now(); + } + } + } + + /** + * Update current check being executed + */ + updateCurrentCheck(testRun: TestRun, checkName: string): void { + const key = this.getKey(testRun); + const state = this.states.get(key); + if (state) { + state.currentCheck = checkName; + } + } + + /** + * Update check result + */ + updateCheckResult(testRun: TestRun, checkResult: CheckResult): void { + const key = this.getKey(testRun); + const state = this.states.get(key); + if (state) { + // Update or add check result + const existingIndex = state.checkResults.findIndex(cr => cr.name === checkResult.name); + if (existingIndex >= 0) { + state.checkResults[existingIndex] = checkResult; + } else { + state.checkResults.push(checkResult); + } + state.currentCheck = undefined; // Clear current check after result + } + } + + /** + * Render the status display + */ + private render(): void { + if (!this.isActive) return; + + const lines: string[] = []; + const colors = { + reset: '\x1b[0m', + bright: '\x1b[1m', + dim: '\x1b[2m', + green: '\x1b[32m', + red: '\x1b[31m', + yellow: '\x1b[33m', + blue: '\x1b[34m', + cyan: '\x1b[36m', + gray: '\x1b[90m', + }; + + // Title + lines.push(`${colors.bright}${colors.cyan}⚡ Test Execution Status${colors.reset}`); + lines.push(''); + + // Group by platform → framework + const byPlatform = this.groupByPlatform(); + + for (const [platform, frameworks] of byPlatform) { + const platformIcon = platform === 'py' ? '🐍' : '📦'; + lines.push(`${platformIcon} ${colors.bright}${platform.toUpperCase()}${colors.reset}`); + + for (const [framework, tests] of frameworks) { + const frameworkStatus = this.getFrameworkStatus(tests); + const frameworkIcon = this.getStatusIcon(frameworkStatus); + lines.push(` ${frameworkIcon} ${framework}`); + + for (const test of tests) { + const testIcon = this.getStatusIcon(test.status); + const executionMode = test.executionMode ? ` ${colors.dim}(${test.executionMode})${colors.reset}` : ''; + const duration = test.startTime ? ` ${colors.gray}${this.formatDuration(Date.now() - test.startTime)}${colors.reset}` : ''; + + lines.push(` ${testIcon} ${test.testName}${executionMode}${duration}`); + + // Show current check if running + if (test.status === 'running' && test.currentCheck) { + lines.push(` ${colors.blue}→${colors.reset} ${colors.dim}${test.currentCheck}...${colors.reset}`); + } + + // Show check results + for (const check of test.checkResults) { + const checkIcon = this.getStatusIcon(check.status); + const checkLine = ` ${checkIcon} ${colors.dim}${check.name}${colors.reset}`; + + if (check.status === 'skipped' && check.skipReason) { + lines.push(`${checkLine} ${colors.gray}(${check.skipReason})${colors.reset}`); + } else if (check.status === 'failed' && check.error) { + lines.push(checkLine); + // Show first line of error + const errorFirstLine = check.error.split('\n')[0]; + lines.push(` ${colors.red}↳${colors.reset} ${colors.dim}${errorFirstLine}${colors.reset}`); + } else { + lines.push(checkLine); + } + } + + // Show error if failed + if (test.status === 'failed' && test.error && test.checkResults.length === 0) { + const errorFirstLine = test.error.split('\n')[0]; + lines.push(` ${colors.red}Error:${colors.reset} ${colors.dim}${errorFirstLine}${colors.reset}`); + } + } + } + lines.push(''); // Blank line between platforms + } + + // Update the terminal output using log-update (no flicker!) + logUpdate(lines.join('\n')); + } + + /** + * Group tests by platform and framework + */ + private groupByPlatform(): Map> { + const result = new Map>(); + + for (const state of this.states.values()) { + if (!result.has(state.platform)) { + result.set(state.platform, new Map()); + } + const frameworks = result.get(state.platform)!; + + if (!frameworks.has(state.framework)) { + frameworks.set(state.framework, []); + } + frameworks.get(state.framework)!.push(state); + } + + return result; + } + + /** + * Get overall status for a framework (all its tests) + */ + private getFrameworkStatus(tests: TestState[]): 'pending' | 'running' | 'passed' | 'failed' | 'skipped' { + if (tests.some(t => t.status === 'failed')) return 'failed'; + if (tests.some(t => t.status === 'running')) return 'running'; + if (tests.every(t => t.status === 'passed')) return 'passed'; + if (tests.every(t => t.status === 'skipped')) return 'skipped'; + return 'pending'; + } + + /** + * Get icon for status + */ + private getStatusIcon(status: string): string { + const colors = { + reset: '\x1b[0m', + green: '\x1b[32m', + red: '\x1b[31m', + yellow: '\x1b[33m', + blue: '\x1b[34m', + gray: '\x1b[90m', + }; + + switch (status) { + case 'passed': + return `${colors.green}✓${colors.reset}`; + case 'failed': + return `${colors.red}✗${colors.reset}`; + case 'skipped': + return `${colors.yellow}⊘${colors.reset}`; + case 'running': + return `${colors.blue}◉${colors.reset}`; + case 'pending': + return `${colors.gray}○${colors.reset}`; + default: + return '·'; + } + } + + /** + * Format duration in seconds + */ + private formatDuration(ms: number): string { + const seconds = (ms / 1000).toFixed(1); + return `${seconds}s`; + } + + /** + * Generate unique key for a test run + */ + private getKey(testRun: TestRun): string { + return testRun.id; + } +} diff --git a/src/runner/framework-config.ts b/src/runner/framework-config.ts new file mode 100644 index 0000000..8531838 --- /dev/null +++ b/src/runner/framework-config.ts @@ -0,0 +1,104 @@ +/** + * Framework configuration schema + */ + +export interface FrameworkConfig { + /** Framework identifier (e.g., "openai", "openai-agents") */ + name: string; + + /** Human-readable display name */ + displayName: string; + + /** Framework type: llm-only or agentic */ + type: 'llm-only' | 'agentic'; + + /** Platform: JavaScript or Python */ + platform: 'js' | 'py'; + + /** Package dependencies to install */ + dependencies: FrameworkDependency[]; + + /** Common versions to test */ + versions: string[]; + + /** Sentry SDK versions to test against */ + sentryVersions: string[]; + + /** Python only: execution mode for the framework */ + executionMode?: 'sync' | 'async' | 'both'; + + /** Streaming mode: whether the framework supports streaming responses */ + streamingMode?: 'streaming' | 'blocking' | 'both'; + + /** Model overrides: Some frameworks use different models than requested */ + modelOverrides?: { + request?: string; + response?: string; + }; + + /** Skip configuration: Tests or checks that should be skipped */ + skip?: { + tests?: string[]; // Array of test names to skip entirely + checks?: { // Per-test check skipping + [testName: string]: string[]; // Array of check method names to skip + }; + }; + + /** Optional: Additional test matrix axes */ + matrix?: { + /** Model providers to test (e.g., ["openai", "anthropic"]) */ + modelProviders?: string[]; + + /** Additional custom axes */ + [key: string]: string[] | undefined; + }; +} + +export interface FrameworkDependency { + /** Package name */ + package: string; + + /** Version (or "latest", "framework" to match framework version) */ + version: string; +} + +/** + * Load framework configuration from JSON file + */ +export function loadFrameworkConfig(configPath: string): FrameworkConfig { + const fs = require('fs'); + + if (!fs.existsSync(configPath)) { + throw new Error(`Framework config not found: ${configPath}`); + } + + try { + const content = fs.readFileSync(configPath, 'utf-8'); + const config = JSON.parse(content) as FrameworkConfig; + + // Validate required fields + const requiredFields = ['name', 'displayName', 'type', 'platform', 'dependencies', 'versions', 'sentryVersions']; + for (const field of requiredFields) { + if (!(field in config)) { + throw new Error(`Missing required field: ${field}`); + } + } + + // Validate type field + if (config.type !== 'llm-only' && config.type !== 'agentic') { + throw new Error(`Invalid type: ${config.type}. Must be 'llm-only' or 'agentic'`); + } + + // Validate platform field + if (config.platform !== 'js' && config.platform !== 'py') { + throw new Error(`Invalid platform: ${config.platform}. Must be 'js' or 'py'`); + } + + return config; + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error(`Invalid JSON in ${configPath}: ${error.message}`); + } + throw error; + } +} diff --git a/src/runner/framework-discovery.ts b/src/runner/framework-discovery.ts new file mode 100644 index 0000000..cf9435c --- /dev/null +++ b/src/runner/framework-discovery.ts @@ -0,0 +1,224 @@ +/** + * Framework Discovery System + * + * Scans the templates directory and loads framework configurations. + */ + +import * as fs from "fs"; +import * as path from "path"; +import { fileURLToPath } from "url"; +import { FrameworkConfig } from "./framework-config.js"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const TEMPLATES_DIR = path.join(__dirname, "templates"); + +/** + * Framework metadata including template path + */ +export interface DiscoveredFramework extends FrameworkConfig { + /** Path to template file */ + templatePath: string; + + /** Category (llm, agents, etc.) */ + category: string; +} + +/** + * Discover all frameworks by scanning templates directory + * + * Directory structure: + * templates/ + * {category}/ # llm, agents, etc. + * {platform}/ # js, py + * {framework}/ # openai, anthropic, etc. + * config.json + * template.njk + */ +export function discoverFrameworks(): DiscoveredFramework[] { + const frameworks: DiscoveredFramework[] = []; + + if (!fs.existsSync(TEMPLATES_DIR)) { + throw new Error(`Templates directory not found: ${TEMPLATES_DIR}`); + } + + // Scan categories (llm, agents, etc.) + const categories = fs + .readdirSync(TEMPLATES_DIR, { withFileTypes: true }) + .filter((dirent) => dirent.isDirectory() && !dirent.name.startsWith(".")) + .map((dirent) => dirent.name); + + for (const category of categories) { + const categoryPath = path.join(TEMPLATES_DIR, category); + + // Scan platforms (js, py) + const platforms = fs + .readdirSync(categoryPath, { withFileTypes: true }) + .filter( + (dirent) => + dirent.isDirectory() && + (dirent.name === "js" || dirent.name === "py"), + ) + .map((dirent) => dirent.name); + + for (const platform of platforms) { + const platformPath = path.join(categoryPath, platform); + + // Scan framework directories + const frameworkDirs = fs + .readdirSync(platformPath, { withFileTypes: true }) + .filter( + (dirent) => dirent.isDirectory() && !dirent.name.startsWith("."), + ) + .map((dirent) => dirent.name); + + for (const frameworkDir of frameworkDirs) { + const frameworkPath = path.join(platformPath, frameworkDir); + const configPath = path.join(frameworkPath, "config.json"); + const templatePath = path.join(frameworkPath, "template.njk"); + + // Validate required files exist + if (!fs.existsSync(configPath)) { + console.warn(`Warning: Missing config.json in ${frameworkPath}`); + continue; + } + + if (!fs.existsSync(templatePath)) { + console.warn(`Warning: Missing template.njk in ${frameworkPath}`); + continue; + } + + // Load and validate config + try { + const configContent = fs.readFileSync(configPath, "utf-8"); + const config = JSON.parse(configContent) as FrameworkConfig; + + // Validate required fields + if ( + !config.name || + !config.displayName || + !config.type || + !config.platform + ) { + console.warn( + `Warning: Invalid config in ${configPath} - missing required fields`, + ); + continue; + } + + // Validate required arrays + if (!config.versions || config.versions.length === 0) { + console.warn( + `Warning: Invalid config in ${configPath} - missing or empty 'versions' array`, + ); + continue; + } + if (!config.sentryVersions || config.sentryVersions.length === 0) { + console.warn( + `Warning: Invalid config in ${configPath} - missing or empty 'sentryVersions' array`, + ); + continue; + } + + // Validate platform matches directory structure + if (config.platform !== platform) { + console.warn( + `Warning: Platform mismatch in ${configPath} - expected ${platform}, got ${config.platform}`, + ); + continue; + } + + // Add discovered framework + frameworks.push({ + ...config, + templatePath, + category, + }); + } catch (error) { + console.warn( + `Warning: Failed to load config from ${configPath}:`, + error, + ); + continue; + } + } + } + } + + return frameworks; +} + +/** + * Get frameworks by category (llm, agents, etc.) + */ +export function getFrameworksByCategory( + category: string, +): DiscoveredFramework[] { + return discoverFrameworks().filter((f) => f.category === category); +} + +/** + * Get frameworks by platform (js, py) + */ +export function getFrameworksByPlatform( + platform: "js" | "py", +): DiscoveredFramework[] { + return discoverFrameworks().filter((f) => f.platform === platform); +} + +/** + * Get frameworks by type (llm-only, agentic) + */ +export function getFrameworksByType( + type: "llm-only" | "agentic", +): DiscoveredFramework[] { + return discoverFrameworks().filter((f) => f.type === type); +} + +/** + * Get a specific framework by name + */ +export function getFrameworkByName( + name: string, +): DiscoveredFramework | undefined { + return discoverFrameworks().find((f) => f.name === name); +} + +/** + * List all available frameworks (for CLI display) + */ +export function listFrameworks(): void { + const frameworks = discoverFrameworks(); + + if (frameworks.length === 0) { + console.log("No frameworks discovered."); + return; + } + + console.log(`\nDiscovered ${frameworks.length} framework(s):\n`); + + // Group by category + const byCategory = frameworks.reduce( + (acc, f) => { + if (!acc[f.category]) acc[f.category] = []; + acc[f.category].push(f); + return acc; + }, + {} as Record, + ); + + for (const [category, categoryFrameworks] of Object.entries(byCategory)) { + console.log(`${category.toUpperCase()}:`); + + for (const framework of categoryFrameworks) { + const platformIcon = framework.platform === "js" ? "🟨" : "🐍"; + const typeIcon = framework.type === "agentic" ? "🤖" : "💬"; + + console.log(` ${platformIcon} ${typeIcon} ${framework.name}`); + console.log(` ${framework.displayName}`); + console.log(` Versions: ${framework.versions.join(", ")}`); + console.log(` Sentry: ${framework.sentryVersions.join(", ")}`); + console.log(); + } + } +} diff --git a/src/runner/javascript-runner.ts b/src/runner/javascript-runner.ts new file mode 100644 index 0000000..ebe3425 --- /dev/null +++ b/src/runner/javascript-runner.ts @@ -0,0 +1,262 @@ +/** + * JavaScript-specific test runner + * Handles Node.js environment setup, dependency installation, and test execution + */ + +import * as path from "path"; +import * as fs from "fs/promises"; +import { exec } from "child_process"; +import { promisify } from "util"; +import { RunnerContext } from "../types.js"; + +const execAsync = promisify(exec); + +export class JavaScriptRunner { + /** + * Check if JavaScript environment needs setup + */ + async needsSetup(workDir: string): Promise { + const nodeModulesPath = path.join(workDir, "node_modules"); + try { + await fs.access(nodeModulesPath); + return false; + } catch { + return true; + } + } + + /** + * Setup Node.js environment and install dependencies + */ + async setupEnvironment(context: RunnerContext): Promise { + const { workDir, framework } = context; + const verbose = context.verbose === true; + + if (verbose) { + console.log(` Setting up Node.js environment in ${workDir}...`); + } + + // Create package.json + const packageJson = this.generatePackageJson(context); + await fs.writeFile( + path.join(workDir, "package.json"), + JSON.stringify(packageJson, null, 2), + ); + if (verbose) { + console.log(" ✓ package.json generated"); + } + + // Install dependencies + if (verbose) { + console.log(" Installing dependencies..."); + } + + // Check for local Sentry SDK path + const localSentryPath = process.env.SENTRY_JAVASCRIPT_PATH; + if (localSentryPath && framework.sentryVersion === "local") { + if (verbose) { + console.log(` Linking local Sentry SDK from: ${localSentryPath}`); + } + // Install other dependencies first + await execAsync("npm install --no-save", { + cwd: workDir, + env: { ...process.env, npm_config_loglevel: "error" }, + }); + // Link local Sentry SDK + await execAsync(`npm link "${localSentryPath}/packages/node"`, { + cwd: workDir, + env: { ...process.env, npm_config_loglevel: "error" }, + }); + } else { + await execAsync("npm install --no-save", { + cwd: workDir, + env: { ...process.env, npm_config_loglevel: "error" }, + }); + } + + if (verbose) { + console.log(" ✓ Dependencies installed"); + } + } + + /** + * Generate package.json content + */ + private generatePackageJson(context: RunnerContext): object { + const { framework } = context; + const dependencies: Record = {}; + + // Add Sentry SDK (skip if using local version, will be linked separately) + if (framework.sentryVersion !== "local") { + dependencies["@sentry/node"] = framework.sentryVersion; + } + + // Add framework dependencies from config + if (framework.dependencies && framework.dependencies.length > 0) { + for (const dep of framework.dependencies) { + let version = dep.version; + + // Replace "framework" with actual framework version + if (version === "framework") { + version = framework.version; + } + + dependencies[dep.package] = version; + } + } else { + // Fallback: Add framework package with framework version + dependencies[framework.name] = framework.version; + } + + return { + name: `test-${framework.name}`, + version: "1.0.0", + type: "module", + dependencies, + }; + } + + /** + * Generate test case ID from test name + */ + private generateTestCaseId(testName: string): string { + return testName + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); + } + + /** + * Execute JavaScript test + */ + async executeTest(context: RunnerContext): Promise { + const { + workDir, + sentryDsn, + runId, + testDefinition, + framework, + isStreaming, + } = context; + const verbose = context.verbose === true; + + if (verbose) { + console.log(" Executing JavaScript test..."); + } + + // Generate test case ID and determine filename (must match runner.ts logic) + const testCaseId = this.generateTestCaseId(testDefinition.name); + const modeParts: string[] = []; + if (framework.streamingMode) { + modeParts.push(isStreaming ? "streaming" : "blocking"); + } + const modeSuffix = modeParts.length > 0 ? `-${modeParts.join("-")}` : ""; + const testFile = path.join(workDir, `test-${testCaseId}${modeSuffix}.js`); + const logFile = path.join(workDir, `test-${testCaseId}${modeSuffix}.log`); + + const env = { + ...process.env, + SENTRY_DSN: sentryDsn, + RUN_ID: runId, + OPENAI_API_KEY: process.env.OPENAI_API_KEY || "", + ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY || "", + GOOGLE_API_KEY: process.env.GOOGLE_API_KEY || "", + GOOGLE_GENAI_API_KEY: process.env.GOOGLE_GENAI_API_KEY || "", + GOOGLE_APPLICATION_CREDENTIALS: + process.env.GOOGLE_APPLICATION_CREDENTIALS || "", + GOOGLE_VERTEX_PROJECT: process.env.GOOGLE_VERTEX_PROJECT || "", + GOOGLE_VERTEX_LOCATION: process.env.GOOGLE_VERTEX_LOCATION || "", + }; + + try { + const { stdout, stderr } = await execAsync(`node ${testFile}`, { + cwd: workDir, + env, + timeout: 60000, // 60 second timeout + }); + + // Write stdout and stderr to log file + const logContent = [ + "=== Test Execution Log ===", + `Test: ${testDefinition.name}`, + `Timestamp: ${new Date().toISOString()}`, + "", + "=== STDOUT ===", + stdout || "(no output)", + "", + "=== STDERR ===", + stderr || "(no errors)", + "", + "=== End of Log ===", + ].join("\n"); + + await fs.writeFile(logFile, logContent, "utf-8"); + + if (verbose) { + console.log(` Log written to: ${path.basename(logFile)}`); + + if (stdout) { + console.log(" Test output:"); + stdout.split("\n").forEach((line) => { + if (line.trim()) console.log(` ${line}`); + }); + } + + if (stderr) { + console.error(" Test errors:"); + stderr.split("\n").forEach((line) => { + if (line.trim()) console.error(` ${line}`); + }); + } + } + } catch (error: any) { + // Write error to log file even on failure + const errorContent = [ + "=== Test Execution Log (FAILED) ===", + `Test: ${testDefinition.name}`, + `Timestamp: ${new Date().toISOString()}`, + "", + "=== ERROR ===", + error.message || "Unknown error", + "", + "=== STDOUT ===", + error.stdout || "(no output)", + "", + "=== STDERR ===", + error.stderr || "(no errors)", + "", + "=== End of Log ===", + ].join("\n"); + + try { + await fs.writeFile(logFile, errorContent, "utf-8"); + if (verbose) { + console.log(` Log written to: ${path.basename(logFile)}`); + } + } catch (writeError) { + if (verbose) { + console.error(" Failed to write log file:", writeError); + } + } + + if (error.code === "ETIMEDOUT") { + throw new Error("Test execution timed out (60s)"); + } + throw new Error( + `Test execution failed: ${error.message}\n${error.stderr || ""}`, + ); + } + } + + /** + * Get Node.js version + */ + async getNodeVersion(): Promise { + try { + const { stdout } = await execAsync("node --version"); + return stdout.trim(); + } catch { + return "Unknown"; + } + } +} diff --git a/src/runner/python-runner.ts b/src/runner/python-runner.ts new file mode 100644 index 0000000..bd639b7 --- /dev/null +++ b/src/runner/python-runner.ts @@ -0,0 +1,357 @@ +/** + * Python-specific test runner + * Handles Python environment setup, dependency installation, and test execution + */ + +import * as path from 'path'; +import * as fs from 'fs/promises'; +import { exec } from 'child_process'; +import { promisify } from 'util'; +import { RunnerContext } from '../types.js'; + +const execAsync = promisify(exec); + +export class PythonRunner { + /** + * Check if Python environment needs setup + */ + async needsSetup(workDir: string): Promise { + const venvPath = path.join(workDir, '.venv'); + try { + await fs.access(venvPath); + return false; + } catch { + return true; + } + } + + /** + * Setup Python virtual environment and install dependencies + */ + async setupEnvironment(context: RunnerContext): Promise { + const { workDir, framework } = context; + const verbose = context.verbose === true; + + if (verbose) { + console.log(` Setting up Python environment in ${workDir}...`); + } + + // Check if uv is available + const useUv = await this.isUvAvailable(); + + if (useUv) { + if (verbose) { + console.log(' Using uv for dependency management'); + } + + // Create pyproject.toml + const pyproject = this.generatePyprojectToml(context); + await fs.writeFile(path.join(workDir, 'pyproject.toml'), pyproject); + if (verbose) { + console.log(' ✓ pyproject.toml generated'); + } + + // Create virtual environment with uv + await execAsync('uv venv .venv', { cwd: workDir }); + if (verbose) { + console.log(' ✓ Virtual environment created'); + } + + // Install dependencies with uv + if (verbose) { + console.log(' Installing dependencies...'); + } + + // Check for local Sentry SDK path + const localSentryPath = process.env.SENTRY_PYTHON_PATH; + if (localSentryPath && framework.sentryVersion === 'local') { + if (verbose) { + console.log(` Installing local Sentry SDK from: ${localSentryPath}`); + } + await execAsync(`uv pip install -e "${localSentryPath}"`, { + cwd: workDir, + env: { ...process.env, VIRTUAL_ENV: path.join(workDir, '.venv') } + }); + } else { + // Install Sentry SDK from PyPI + await execAsync(`uv pip install sentry-sdk==${framework.sentryVersion}`, { + cwd: workDir, + env: { ...process.env, VIRTUAL_ENV: path.join(workDir, '.venv') } + }); + } + + // Install project dependencies from pyproject.toml + await execAsync('uv pip install .', { + cwd: workDir, + env: { ...process.env, VIRTUAL_ENV: path.join(workDir, '.venv') } + }); + + if (verbose) { + console.log(' ✓ Dependencies installed'); + } + } else { + // Fallback to traditional pip-based approach + if (verbose) { + console.log(' Using pip for dependency management'); + } + + // Create virtual environment + await execAsync('python3 -m venv .venv', { cwd: workDir }); + if (verbose) { + console.log(' ✓ Virtual environment created'); + } + + const pipPath = path.join(workDir, '.venv', 'bin', 'pip'); + await execAsync(`${pipPath} install --upgrade pip`, { cwd: workDir }); + + // Check for local Sentry SDK path + const localSentryPath = process.env.SENTRY_PYTHON_PATH; + if (localSentryPath && framework.sentryVersion === 'local') { + if (verbose) { + console.log(` Installing local Sentry SDK from: ${localSentryPath}`); + } + await execAsync(`${pipPath} install -e "${localSentryPath}"`, { cwd: workDir }); + } else { + await execAsync(`${pipPath} install sentry-sdk==${framework.sentryVersion}`, { cwd: workDir }); + } + + // Create requirements.txt for other dependencies + const requirements = this.generateRequirements(context); + await fs.writeFile(path.join(workDir, 'requirements.txt'), requirements); + if (verbose) { + console.log(' ✓ requirements.txt generated'); + } + + // Install other dependencies + if (verbose) { + console.log(' Installing dependencies...'); + } + await execAsync(`${pipPath} install -r requirements.txt`, { + cwd: workDir, + env: { ...process.env, PIP_NO_CACHE_DIR: '1' } + }); + if (verbose) { + console.log(' ✓ Dependencies installed'); + } + } + } + + /** + * Check if uv is available + */ + private async isUvAvailable(): Promise { + try { + await execAsync('uv --version'); + return true; + } catch { + return false; + } + } + + /** + * Generate pyproject.toml content + */ + private generatePyprojectToml(context: RunnerContext): string { + const { framework } = context; + + const dependencies: string[] = []; + + // Add framework dependencies from config + if (framework.dependencies && framework.dependencies.length > 0) { + for (const dep of framework.dependencies) { + let version = dep.version; + + // Replace "framework" with actual framework version + if (version === 'framework') { + version = framework.version; + } + + // Add version specifier + if (version === 'latest') { + dependencies.push(`"${dep.package}"`); + } else { + dependencies.push(`"${dep.package}==${version}"`); + } + } + } else { + // Fallback: Add framework package with framework version + dependencies.push(`"${framework.name}==${framework.version}"`); + } + + return `[project] +name = "sentry-test-${framework.name}" +version = "0.1.0" +requires-python = ">=3.9" +dependencies = [ +${dependencies.map(d => ` ${d},`).join('\n')} +] +`; + } + + /** + * Generate requirements.txt content (for pip fallback, without sentry-sdk) + */ + private generateRequirements(context: RunnerContext): string { + const { framework } = context; + const requirements: string[] = []; + + // Add framework dependencies from config (but NOT sentry-sdk, handled separately) + if (framework.dependencies && framework.dependencies.length > 0) { + for (const dep of framework.dependencies) { + let version = dep.version; + + // Replace "framework" with actual framework version + if (version === 'framework') { + version = framework.version; + } + + // Add version specifier if not "latest" + if (version === 'latest') { + requirements.push(dep.package); + } else { + requirements.push(`${dep.package}==${version}`); + } + } + } else { + // Fallback: Add framework package with framework version + requirements.push(`${framework.name}==${framework.version}`); + } + + return requirements.join('\n'); + } + + /** + * Generate test case ID from test name + */ + private generateTestCaseId(testName: string): string { + return testName + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); + } + + /** + * Execute Python test + */ + async executeTest(context: RunnerContext): Promise { + const { workDir, sentryDsn, runId, isAsync, isStreaming, testDefinition, framework } = context; + const verbose = context.verbose === true; + + if (verbose) { + console.log(' Executing Python test...'); + } + + const pythonPath = path.join(workDir, '.venv', 'bin', 'python'); + + // Generate test case ID and determine filename (must match runner.ts logic) + const testCaseId = this.generateTestCaseId(testDefinition.name); + const modeParts: string[] = []; + modeParts.push(isAsync ? 'async' : 'sync'); + if (framework.streamingMode) { + modeParts.push(isStreaming ? 'streaming' : 'blocking'); + } + const modeSuffix = modeParts.join('-'); + const testFile = path.join(workDir, `test-${testCaseId}-${modeSuffix}.py`); + const logFile = path.join(workDir, `test-${testCaseId}-${modeSuffix}.log`); + + const env = { + ...process.env, + SENTRY_DSN: sentryDsn, + RUN_ID: runId, + OPENAI_API_KEY: process.env.OPENAI_API_KEY || '', + ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY || '', + GOOGLE_API_KEY: process.env.GOOGLE_API_KEY || '', + }; + + try { + const { stdout, stderr } = await execAsync(`${pythonPath} ${testFile}`, { + cwd: workDir, + env, + timeout: 60000, // 60 second timeout + }); + + // Write stdout and stderr to log file + const logContent = [ + '=== Test Execution Log ===', + `Timestamp: ${new Date().toISOString()}`, + `Test: ${testDefinition.name}`, + `Framework: ${context.framework.name}`, + `Mode: ${modeSuffix}`, + '', + '=== STDOUT ===', + stdout, + '', + '=== STDERR ===', + stderr, + ].join('\n'); + + await fs.writeFile(logFile, logContent); + + if (verbose) { + console.log(` Log written to: ${path.basename(logFile)}`); + + if (stdout.trim()) { + console.log(' Test output:'); + for (const line of stdout.split('\n')) { + if (line.trim()) console.log(` ${line}`); + } + } + + if (stderr) { + console.error(' Test errors:'); + stderr.split('\n').forEach(line => { + if (line.trim()) console.error(` ${line}`); + }); + } + } + } catch (error: any) { + // Write error to log file even on failure + const errorContent = [ + '=== Test Execution Failed ===', + `Timestamp: ${new Date().toISOString()}`, + `Test: ${testDefinition.name}`, + `Framework: ${context.framework.name}`, + `Mode: ${modeSuffix}`, + '', + '=== STDOUT ===', + error.stdout || '', + '', + '=== STDERR ===', + error.stderr || '', + '', + '=== ERROR ===', + error.message, + ].join('\n'); + + try { + await fs.writeFile(logFile, errorContent); + + if (verbose) { + console.log(` Log written to: ${path.basename(logFile)}`); + } + } catch (writeError) { + if (verbose) { + console.error(' Failed to write log file:', writeError); + } + } + + if (error.code === 'ETIMEDOUT') { + throw new Error('Test execution timed out (60s)'); + } + throw new Error(`Test execution failed: ${error.message}\n${error.stderr || ''}`); + } + } + + /** + * Get Python version + */ + async getPythonVersion(workDir: string): Promise { + const pythonPath = path.join(workDir, '.venv', 'bin', 'python'); + try { + const { stdout } = await execAsync(`${pythonPath} --version`); + return stdout.trim(); + } catch { + return 'Unknown'; + } + } +} diff --git a/src/runner/runner.ts b/src/runner/runner.ts new file mode 100644 index 0000000..f9bb69b --- /dev/null +++ b/src/runner/runner.ts @@ -0,0 +1,257 @@ +/** + * Runner - orchestrates template rendering and test execution + */ + +import { RunnerContext, FrameworkConfig } from "../types.js"; +import { TemplateRenderer } from "./template-renderer.js"; +import { PythonRunner } from "./python-runner.js"; +import { JavaScriptRunner } from "./javascript-runner.js"; +import * as path from "path"; +import * as fs from "fs/promises"; +import * as prettier from "prettier"; + +export class Runner { + private runsDir: string; + private renderer: TemplateRenderer; + private pythonRunner: PythonRunner; + private jsRunner: JavaScriptRunner; + + constructor() { + this.runsDir = path.join(process.cwd(), "runs"); + this.renderer = new TemplateRenderer(); + this.pythonRunner = new PythonRunner(); + this.jsRunner = new JavaScriptRunner(); + } + + /** + * Get work directory for a framework + */ + getWorkDir(framework: FrameworkConfig): string { + const { platform, name, version, sentryVersion } = framework; + return path.join( + this.runsDir, + platform, + `${name}-${version}-sentry-${sentryVersion}`, + ); + } + + /** + * Run a test + */ + async runTest(context: RunnerContext): Promise { + const workDir = context.workDir; + const verbose = context.verbose !== false; // Default to true + + // Ensure work directory exists + await fs.mkdir(workDir, { recursive: true }); + + // Get platform-specific runner + const platformRunner = + context.framework.platform === "py" ? this.pythonRunner : this.jsRunner; + + // Check if environment needs setup + const needsSetup = await platformRunner.needsSetup(workDir); + if (needsSetup) { + await platformRunner.setupEnvironment(context); + } else if (verbose) { + console.log(" Using cached environment"); + } + + // Render template + await this.renderTemplate(context); + + // Execute test + await platformRunner.executeTest(context); + } + + /** + * Setup only (environment + template) without executing the test + */ + async setupOnly(context: RunnerContext): Promise { + const workDir = context.workDir; + const verbose = context.verbose !== false; + + // Ensure work directory exists + await fs.mkdir(workDir, { recursive: true }); + + // Get platform-specific runner + const platformRunner = + context.framework.platform === "py" ? this.pythonRunner : this.jsRunner; + + // Check if environment needs setup + const needsSetup = await platformRunner.needsSetup(workDir); + if (needsSetup) { + await platformRunner.setupEnvironment(context); + } else if (verbose) { + console.log(" Using cached environment"); + } + + // Render template + await this.renderTemplate(context); + } + + /** + * Setup environment only (no template rendering) + * Used when we want to setup environments for all tests first, then render templates + */ + async setupEnvironmentOnly(context: RunnerContext): Promise { + const workDir = context.workDir; + const verbose = context.verbose !== false; + + // Ensure work directory exists + await fs.mkdir(workDir, { recursive: true }); + + // Get platform-specific runner + const platformRunner = + context.framework.platform === "py" ? this.pythonRunner : this.jsRunner; + + // Check if environment needs setup + const needsSetup = await platformRunner.needsSetup(workDir); + if (needsSetup) { + await platformRunner.setupEnvironment(context); + } else if (verbose) { + console.log(" Using cached environment"); + } + } + + /** + * Render template only (assumes environment is already set up) + */ + async renderTemplateOnly(context: RunnerContext): Promise { + const workDir = context.workDir; + + // Ensure work directory exists + await fs.mkdir(workDir, { recursive: true }); + + // Render template and return the path + return await this.renderTemplate(context); + } + + /** + * Execute test only (assumes template is already rendered) + */ + async executeOnly(context: RunnerContext): Promise { + // Get platform-specific runner + const platformRunner = + context.framework.platform === "py" ? this.pythonRunner : this.jsRunner; + + // Execute test + await platformRunner.executeTest(context); + } + + /** + * Generate test case ID from test name + * Converts "Basic LLM Test" to "basic-llm-test" + */ + private generateTestCaseId(testName: string): string { + return testName + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); + } + + /** + * Render template and return the test file path + */ + private async renderTemplate(context: RunnerContext): Promise { + const verbose = context.verbose !== false; // Default to true + if (verbose) { + console.log(` Rendering template for ${context.framework.name}...`); + } + + const { workDir, framework, testDefinition, isAsync, isStreaming } = + context; + + // Generate test case ID from test name + const testCaseId = this.generateTestCaseId(testDefinition.name); + + // Build mode suffix for filename + const modeParts: string[] = []; + if (framework.platform === "py") { + modeParts.push(isAsync ? "async" : "sync"); + } + if (framework.streamingMode) { + modeParts.push(isStreaming ? "streaming" : "blocking"); + } + + // Determine test filename based on platform and modes + const extension = framework.platform === "js" ? "js" : "py"; + const modeSuffix = modeParts.length > 0 ? `-${modeParts.join("-")}` : ""; + const testFile = `test-${testCaseId}${modeSuffix}.${extension}`; + + const testPath = path.join(workDir, testFile); + + // Apply model overrides to inputs if specified in framework config + let processedInputs = testDefinition.inputs; + if (framework.modelOverrides) { + processedInputs = testDefinition.inputs.map((input) => ({ + ...input, + model: framework.modelOverrides?.request || input.model, + })); + } + + // Build template context + const templateContext = { + testName: testDefinition.name, + frameworkName: framework.name, + sentryDsn: context.sentryDsn, + runId: context.runId, + isAsync: isAsync || false, // Boolean flag for templates + isStreaming: isStreaming || false, // Boolean flag for streaming mode + causeAPIError: testDefinition.causeAPIError || false, // Flag to intentionally cause API errors + ...(testDefinition.agent && { agent: testDefinition.agent }), + inputs: processedInputs, + }; + + // Render framework template if available, otherwise base template + let rendered: string; + if (framework.category && framework.templatePath) { + // Use discovered framework template + rendered = this.renderer.renderFramework( + framework.category as "llm" | "agents", + framework.platform, + framework.name, + templateContext, + ); + } else { + // Fallback to base template + rendered = this.renderer.renderBase(framework.platform, templateContext); + } + + await fs.writeFile(testPath, rendered); + + // Format the rendered file + await this.formatFile(testPath, framework.platform); + + return testPath; + } + + /** + * Format a generated test file + * Uses Prettier JS API for JavaScript, black CLI for Python + */ + private async formatFile( + filePath: string, + platform: "js" | "py", + ): Promise { + try { + if (platform === "py") { + // Python formatting requires black CLI (optional) + const { exec } = await import("child_process"); + const { promisify } = await import("util"); + const execAsync = promisify(exec); + await execAsync(`black --quiet "${filePath}"`, { timeout: 10000 }); + } else { + // Use Prettier JS API for JavaScript + const source = await fs.readFile(filePath, "utf-8"); + const formatted = await prettier.format(source, { + filepath: filePath, + parser: "babel", + }); + await fs.writeFile(filePath, formatted); + } + } catch { + // Formatting failed silently - not critical + } + } +} diff --git a/src/runner/template-renderer.ts b/src/runner/template-renderer.ts new file mode 100644 index 0000000..371ca42 --- /dev/null +++ b/src/runner/template-renderer.ts @@ -0,0 +1,121 @@ +/** + * Template renderer using Nunjucks + */ + +import nunjucks from 'nunjucks'; +import * as path from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +export interface TemplateContext { + testName: string; + frameworkName: string; + [key: string]: any; +} + +export class TemplateRenderer { + private env: nunjucks.Environment; + private templatesDir: string; + + constructor() { + this.templatesDir = path.join(__dirname, 'templates'); + + // Configure Nunjucks + this.env = nunjucks.configure(this.templatesDir, { + autoescape: false, // Don't escape code + trimBlocks: true, + lstripBlocks: true, + }); + + // Add custom filters + this.env.addFilter('tojson', (obj) => { + return JSON.stringify(obj, null, 2); + }); + } + + /** + * Render a template + * + * @param templatePath - Path relative to templates dir (e.g., 'llm/py/openai/template.njk') + */ + render(templatePath: string, context: TemplateContext): string { + return this.env.render(templatePath, context); + } + + /** + * Render a framework template + * + * @param type - 'llm' or 'agents' + * @param platform - 'js' or 'py' + * @param frameworkName - Framework name (e.g., 'openai') + * @param context - Template context + */ + renderFramework( + type: 'llm' | 'agents', + platform: 'js' | 'py', + frameworkName: string, + context: TemplateContext + ): string { + const templatePath = `${type}/${platform}/${frameworkName}/template.njk`; + return this.render(templatePath, context); + } + + /** + * Render base template for a platform + */ + renderBase(platform: 'js' | 'py', context: TemplateContext): string { + const templateFile = platform === 'js' ? 'base.js.njk' : 'base.py.njk'; + return this.render(templateFile, context); + } + + /** + * Render template string (for inline templates) + */ + renderString(template: string, context: TemplateContext): string { + return nunjucks.renderString(template, context); + } + + /** + * Extend a base template with custom blocks + */ + renderWithBlocks( + platform: 'js' | 'py', + context: TemplateContext, + blocks: { + imports?: string; + sdk_setup?: string; + setup?: string; + test?: string; + teardown?: string; + } + ): string { + // Build template that extends base + const baseTemplate = platform === 'js' ? 'base.js.njk' : 'base.py.njk'; + const extension = platform === 'js' ? '.js' : '.py'; + + let template = `{% extends "${baseTemplate}" %}\n\n`; + + if (blocks.imports) { + template += `{% block imports %}\n{{ super() }}\n${blocks.imports}\n{% endblock %}\n\n`; + } + + if (blocks.sdk_setup) { + template += `{% block sdk_setup %}\n${blocks.sdk_setup}\n{% endblock %}\n\n`; + } + + if (blocks.setup) { + template += `{% block setup %}\n${blocks.setup}\n{% endblock %}\n\n`; + } + + if (blocks.test) { + template += `{% block test %}\n${blocks.test}\n{% endblock %}\n\n`; + } + + if (blocks.teardown) { + template += `{% block teardown %}\n${blocks.teardown}\n{% endblock %}\n\n`; + } + + return this.renderString(template, context); + } +} diff --git a/src/runner/templates/README.md b/src/runner/templates/README.md new file mode 100644 index 0000000..c4eb04d --- /dev/null +++ b/src/runner/templates/README.md @@ -0,0 +1,231 @@ +# Test Templates + +Base templates and framework-specific templates for generating test files using Nunjucks. + +## Directory Structure + +``` +templates/ +├── base.js.njk # JavaScript base template +├── base.py.njk # Python base template +├── llm/ # LLM-only framework templates +│ ├── js/ +│ │ ├── openai/ +│ │ │ ├── template.njk +│ │ │ └── config.json +│ │ └── anthropic/ +│ │ ├── template.njk +│ │ └── config.json +│ └── py/ +│ ├── openai/ +│ │ ├── template.njk +│ │ └── config.json +│ └── anthropic/ +│ ├── template.njk +│ └── config.json +└── agents/ # Agentic framework templates + ├── js/ + │ ├── vercel/ + │ │ ├── template.njk + │ │ └── config.json + │ ├── langgraph/ + │ │ ├── template.njk + │ │ └── config.json + │ └── mastra/ + │ ├── template.njk + │ └── config.json + └── py/ + ├── openai-agents/ + │ ├── template.njk + │ └── config.json + ├── langgraph/ + │ ├── template.njk + │ └── config.json + ├── pydantic-ai/ + │ ├── template.njk + │ └── config.json + └── google-genai/ + ├── template.njk + └── config.json +``` + +## Base Templates + +### `base.js.njk` - JavaScript Base Template + +Provides a standard structure for JavaScript tests with the following blocks: + +- **`imports`** - Import statements (includes `@sentry/node` by default) +- **`sdk_setup`** - Sentry SDK initialization +- **`setup`** - Code to run before the test (setup fixtures, clients, etc.) +- **`test`** - Main test logic (inside async `main()` function) +- **`teardown`** - Code to run after the test + +### `base.py.njk` - Python Base Template + +Provides a standard structure for Python tests with the following blocks: + +- **`imports`** - Import statements (includes `sentry_sdk` by default) +- **`sdk_setup`** - Sentry SDK initialization +- **`setup`** - Code to run before the test (setup fixtures, clients, etc.) +- **`test`** - Main test logic (inside `main()` function) +- **`teardown`** - Code to run after the test + +## Usage + +### 1. Render Base Template + +```typescript +const renderer = new TemplateRenderer(); + +const code = renderer.renderBase('js', { + testName: 'Basic LLM Test', + frameworkName: 'openai', +}); +``` + +### 2. Extend with Custom Blocks + +```typescript +const code = renderer.renderWithBlocks('py', { + testName: 'OpenAI Chat Test', + frameworkName: 'openai', +}, { + imports: 'from openai import OpenAI', + setup: 'client = OpenAI()', + test: ` + response = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": "Hello"}] + ) + print(response.choices[0].message.content) + `, +}); +``` + +## Block Inheritance + +Blocks can use `{{ super() }}` to include the parent block's content: + +```nunjucks +{% block imports %} +{{ super() }} {# Includes base imports #} +from openai import OpenAI {# Add custom import #} +{% endblock %} +``` + +## Framework Templates + +Framework-specific templates extend base templates and implement SDK-specific code. + +### Location + +- **LLM frameworks:** `llm/{js,py}/{framework}.njk` +- **Agent frameworks:** `agents/{js,py}/{framework}.njk` + +### Example: OpenAI Python LLM Template + +**File:** `llm/py/openai/template.njk` + +```nunjucks +{% extends "base.py.njk" %} + +{% block imports %} +{{ super() }} +from openai import OpenAI +{% endblock %} + +{% block setup %} +client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY")) +{% endblock %} + +{% block test %} +response = client.chat.completions.create( + model="{{ input.model }}", + messages=[ + {"role": "system", "content": "{{ system }}"}, + {"role": "user", "content": "{{ input.prompt }}"} + ] +) +print(response.choices[0].message.content) +{% endblock %} +``` + +### Example: OpenAI Agents Python Template + +**File:** `agents/py/openai-agents/template.njk` + +```nunjucks +{% extends "base.py.njk" %} + +{% block imports %} +{{ super() }} +from openai import OpenAI +{% endblock %} + +{% block setup %} +client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY")) + +# Define tools +def {{ agent.tools[0].name }}(): + """{{ agent.tools[0].description }}""" + return {{ agent.tools[0].result }} + +tools = [ + { + "type": "function", + "function": { + "name": "{{ agent.tools[0].name }}", + "description": "{{ agent.tools[0].description }}", + "parameters": {{ agent.tools[0].parameters | dump }} + } + } +] +{% endblock %} + +{% block test %} +response = client.chat.completions.create( + model="{{ input.model }}", + messages=[{"role": "user", "content": "{{ input.prompt }}"}], + tools=tools +) +print(response.choices[0].message.content) +{% endblock %} +``` + +### Template Context Variables + +Framework templates receive: + +- `testName` - Test case name +- `frameworkName` - Framework identifier +- `system` - System message (LLM tests only) +- `agent` - Agent config with tools (agent tests only) +- `input` - Test input with model, prompt, etc. +- `input.model` - Model identifier +- `input.prompt` - User prompt + +### Creating Framework Templates + +1. Determine framework type (LLM or agent) +2. Choose platform (js or py) +3. Create framework directory: `{llm,agents}/{js,py}/{framework}/` +4. Create template file: `template.njk` +5. Create config file: `config.json` +6. Extend appropriate base template +7. Override blocks with framework-specific code + +## Context Variables + +All templates receive a context object with: + +- `testName` - Name of the test +- `frameworkName` - Name of the framework being tested +- Additional variables can be passed as needed + +## Notes + +- Templates use Nunjucks syntax (Jinja-like) +- Autoescape is disabled (we're generating code, not HTML) +- `trimBlocks` and `lstripBlocks` are enabled for cleaner output +- Comments use `{# comment #}` syntax diff --git a/src/runner/templates/agents/README.md b/src/runner/templates/agents/README.md new file mode 100644 index 0000000..76cb634 --- /dev/null +++ b/src/runner/templates/agents/README.md @@ -0,0 +1,120 @@ +# Agent Framework Templates + +Templates for frameworks that support agentic workflows with tool calling. + +## Compatible Frameworks + +| Platform | Framework | Directory | Status | +|----------|-----------|-----------|--------| +| JavaScript | Vercel AI SDK | `js/vercel/` | ✅ Done | +| JavaScript | LangGraph | `js/langgraph/` | ✅ Done | +| JavaScript | Mastra | `js/mastra/` | ✅ Done | +| Python | OpenAI Agents | `py/openai-agents/` | ✅ Done | +| Python | LangGraph | `py/langgraph/` | ✅ Done | +| Python | PydanticAI | `py/pydantic-ai/` | ✅ Done | +| Python | Google GenAI | `py/google-genai/` | ✅ Done | + +## Test Compatibility + +Agent templates should implement tests that have: +- `agent` property (agent configuration with tools) +- `agent.name` property (agent name) +- `agent.description` property (agent description) +- `agent.tools` array (tool definitions) +- `input.model` property (model identifier) +- `input.prompt` property (user prompt) + +Example test from `test-cases/agents/basic.ts`: +```typescript +{ + agent: { + name: 'math_assistant', + description: 'A math assistant', + tools: [ + { + name: 'add', + description: 'Add two numbers', + parameters: { /* JSON Schema */ }, + result: 11, // Static result + } + ] + }, + input: { + model: 'gpt-4o', + prompt: 'What is 4 + 7? Use the add tool.', + } +} +``` + +## Template Requirements + +Each agent template must: + +1. **Extend base template** + ```nunjucks + {% extends "base.{js,py}.njk" %} + ``` + +2. **Import SDK** + ```nunjucks + {% block imports %} + {{ super() }} + from framework import Agent, Tool + {% endblock %} + ``` + +3. **Define tools** + ```nunjucks + {% block setup %} + {% for tool in agent.tools %} + def {{ tool.name }}(): + """{{ tool.description }}""" + {% if tool.result %} + return {{ tool.result }} + {% elif tool.error %} + raise Exception("{{ tool.error }}") + {% endif %} + {% endfor %} + + agent = Agent( + name="{{ agent.name }}", + tools=[{{ agent.tools | map(attribute='name') | join(', ') }}] + ) + {% endblock %} + ``` + +4. **Run agent** + ```nunjucks + {% block test %} + result = agent.run("{{ input.prompt }}") + print(result) + {% endblock %} + ``` + +## Tool Definition Format + +Tools in test definitions have: + +```typescript +{ + name: 'function_name', + description: 'What the function does', + parameters: { /* JSON Schema */ }, + result?: any, // Static return value + error?: string, // OR error to raise +} +``` + +Templates should generate tool implementations that: +- Return `result` if specified +- Raise error with `error` message if specified +- Match the parameter schema + +## Notes + +- Use `{{ agent.name }}` for agent name +- Use `{{ agent.description }}` for agent description +- Loop through `{{ agent.tools }}` to define tools +- Use `{{ input.model }}` for model +- Use `{{ input.prompt }}` for user prompt +- Tool results are static (no real computation) diff --git a/src/runner/templates/agents/js/langgraph/config.json b/src/runner/templates/agents/js/langgraph/config.json new file mode 100644 index 0000000..823728e --- /dev/null +++ b/src/runner/templates/agents/js/langgraph/config.json @@ -0,0 +1,21 @@ +{ + "name": "langgraph", + "displayName": "LangGraph JavaScript SDK", + "type": "agentic", + "platform": "js", + "dependencies": [ + { + "package": "@langchain/langgraph", + "version": "framework" + }, + { + "package": "@langchain/openai", + "version": "0.3.0" + }, + { + "package": "@langchain/core", + "version": "0.3.49" + } + ], + "versions": ["0.2.67"] +} diff --git a/src/runner/templates/agents/js/langgraph/template.njk b/src/runner/templates/agents/js/langgraph/template.njk new file mode 100644 index 0000000..7060af8 --- /dev/null +++ b/src/runner/templates/agents/js/langgraph/template.njk @@ -0,0 +1,64 @@ +{% extends "base.js.njk" %} + +{# Macro to render message content for LangChain/LangGraph - handles both string and multimodal array #} +{% macro renderLangChainContent(content) %} +{% if content is string %} +"{{ content }}" +{%- elif content is iterable %} +[ +{% for part in content %} +{% if part.type == 'text' %} + { type: "text", text: "{{ part.text }}" }, +{% elif part.type == 'image' %} + { type: "image_url", image_url: { url: "data:{{ part.mediaType }};base64,{{ part.base64 }}" } }, +{% endif %} +{% endfor %} + ] +{%- endif %} +{% endmacro %} + +{% block setup %} +// Classes and agent will be initialized after dynamic import +let ChatOpenAI, createReactAgent, HumanMessage, SystemMessage; +let llm, agent; +{% endblock %} + +{% block dynamic_imports %} + // Dynamic import of langgraph AFTER Sentry.init() to ensure instrumentation + const langchainOpenAI = await import("@langchain/openai"); + const langgraph = await import("@langchain/langgraph/prebuilt"); + const langchainMessages = await import("@langchain/core/messages"); + ChatOpenAI = langchainOpenAI.ChatOpenAI; + createReactAgent = langgraph.createReactAgent; + HumanMessage = langchainMessages.HumanMessage; + SystemMessage = langchainMessages.SystemMessage; + llm = new ChatOpenAI({ + modelName: {% if causeAPIError %}"invalid-model"{% else %}"{{ inputs[0].model }}"{% endif %}, + }); + agent = createReactAgent({ llm, tools: [] }); +{% endblock %} + +{% block test %} +{% for input in inputs %} + // Request {{ loop.index }}{% if loop.length > 1 %} of {{ loop.length }}{% endif %} + + const messages{{ loop.index }} = [ +{% for message in input.messages %} +{% if message.role == 'system' %} + new SystemMessage("{{ message.content }}"), +{% elif message.role == 'user' %} + new HumanMessage({ content: {{ renderLangChainContent(message.content) }} }), +{% endif %} +{% endfor %} + ]; + + try { + const result = await agent.invoke({ messages: messages{{ loop.index }} }); + const lastMessage = result.messages[result.messages.length - 1]; + console.log("Response:", lastMessage.content); + } catch (error) { + Sentry.captureException(error); + console.error("Error:", error.message); + } +{% endfor %} +{% endblock %} diff --git a/src/runner/templates/agents/js/mastra/config.json b/src/runner/templates/agents/js/mastra/config.json new file mode 100644 index 0000000..e14f3e8 --- /dev/null +++ b/src/runner/templates/agents/js/mastra/config.json @@ -0,0 +1,55 @@ +{ + "name": "mastra", + "displayName": "Mastra AI Framework", + "type": "agentic", + "platform": "js", + "dependencies": [ + { + "package": "@mastra/core", + "version": "framework" + }, + { + "package": "@mastra/sentry", + "version": "beta" + }, + { + "package": "@mastra/observability", + "version": "latest" + } + ], + "versions": ["1.2.0"], + "sentryVersions": ["latest"], + "skip": { + "tests": ["Long Input Agent Test"], + "checks": { + "Basic Agent Test": [ + "checkAgentSpanAttributes", + "checkChatSpanAttributes", + "checkAgentHierarchy" + ], + "Tool Call Agent Test": [ + "checkAgentSpanAttributes", + "checkChatSpanAttributes", + "checkAgentHierarchy", + "checkToolSpanAttributes", + "checkAvailableTools", + "checkResponseToolCalls(add, multiply)", + "checkToolCalls(add, multiply)" + ], + "Tool Error Agent Test": [ + "checkAgentSpanAttributes", + "checkChatSpanAttributes", + "checkAgentHierarchy", + "checkToolSpanAttributes", + "checkAvailableTools", + "checkResponseToolCalls(read_file)" + ], + "Vision Agent Test": [ + "checkAgentSpanAttributes", + "checkChatSpanAttributes", + "checkAgentHierarchy", + "checkBinaryRedaction" + ] + } + } +} diff --git a/src/runner/templates/agents/js/mastra/template.njk b/src/runner/templates/agents/js/mastra/template.njk new file mode 100644 index 0000000..526edde --- /dev/null +++ b/src/runner/templates/agents/js/mastra/template.njk @@ -0,0 +1,166 @@ +/** + * Sentry AI SDK Integration Test: {{ testName }} + * Framework: {{ frameworkName }} + * + * This test verifies that Sentry correctly instruments {{ frameworkName }} SDK calls. + * + * Mastra uses its own @mastra/sentry exporter which sends traces to Sentry + * using OpenTelemetry semantic conventions. This is different from other frameworks + * that use the standard @sentry/node integration. + */ + +// Mastra has its own Sentry integration via @mastra/sentry +// We do NOT import @sentry/node directly - Mastra handles this internally + +{% block setup %} +// Variables will be initialized after dynamic import +let Agent, createTool, Mastra, Observability, SentryExporter, z; +let mastra, agent; +{% endblock %} + +async function main() { +{% block dynamic_imports %} + // Dynamic imports + const mastraCore = await import("@mastra/core"); + const mastraAgent = await import("@mastra/core/agent"); + const mastraTools = await import("@mastra/core/tools"); + const mastraSentry = await import("@mastra/sentry"); + const mastraObservability = await import("@mastra/observability"); + const zod = await import("zod"); + + Agent = mastraAgent.Agent; + createTool = mastraTools.createTool; + Mastra = mastraCore.Mastra; + Observability = mastraObservability.Observability; + SentryExporter = mastraSentry.SentryExporter; + z = zod.z; +{% endblock %} + +{% block tools %} +{% if agent and agent.tools and agent.tools.length > 0 %} + // Define tools + const tools = { +{% for tool in agent.tools %} + {{ tool.name }}: createTool({ + id: "{{ tool.name }}", + description: "{{ tool.description }}", + inputSchema: z.object({ +{% for paramName, paramDef in tool.parameters.properties %} +{% if paramDef.type == 'number' or paramDef.type == 'integer' %} + {{ paramName }}: z.number().describe("{{ paramDef.description | default('') }}"), +{% elif paramDef.type == 'boolean' %} + {{ paramName }}: z.boolean().describe("{{ paramDef.description | default('') }}"), +{% else %} + {{ paramName }}: z.string().describe("{{ paramDef.description | default('') }}"), +{% endif %} +{% endfor %} + }), + outputSchema: z.object({ + result: z.any(), + }), + execute: async (inputData) => { +{% if tool.error %} + throw new Error("{{ tool.error }}"); +{% elif tool.result is defined %} + // Return predefined result + return { result: {{ tool.result | dump }} }; +{% else %} + // Return input for validation + return { result: inputData }; +{% endif %} + }, + }), +{% endfor %} + }; +{% else %} + // No tools defined + const tools = {}; +{% endif %} +{% endblock %} + +{% block agent %} +{% set system_content = "" %} +{% for input in inputs %} +{% for message in input.messages %} +{% if message.role == 'system' %}{% set system_content = message.content %}{% endif %} +{% endfor %} +{% endfor %} + // Create the agent + agent = new Agent({ + id: "{{ agent.name | default('test-agent') }}", + name: "{{ agent.name | default('Test Agent') }}", + instructions: "{{ system_content | default('You are a helpful assistant.') }}", + model: {% if causeAPIError %}"openai/invalid-model"{% else %}"openai/{{ inputs[0].model }}"{% endif %}, +{% if agent and agent.tools and agent.tools.length > 0 %} + tools: tools, +{% endif %} + }); +{% endblock %} + +{% block mastra %} + // Initialize Mastra with Sentry observability + mastra = new Mastra({ + agents: { agent }, + observability: new Observability({ + configs: { + sentry: { + serviceName: "mastra-test", + exporters: [ + new SentryExporter({ + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 1.0, + }), + ], + }, + }, + }), + }); + + // Get the agent from Mastra instance (gives access to observability) + const testAgent = mastra.getAgent("agent"); +{% endblock %} + +{% block test %} +{% for input in inputs %} +{% set user_content = "" %} +{% for message in input.messages %} +{% if message.role == 'user' %}{% set user_content = message.content %}{% endif %} +{% endfor %} + // Request {{ loop.index }}{% if loop.length > 1 %} of {{ loop.length }}{% endif %} + + try { + const response = await testAgent.generate([ +{% for message in input.messages %} +{% if message.role == 'user' %} + { role: "user", content: "{{ message.content }}" }, +{% elif message.role == 'assistant' %} + { role: "assistant", content: "{{ message.content }}" }, +{% endif %} +{% endfor %} + ]{% if agent and agent.tools and agent.tools.length > 0 %}, { maxSteps: 10 }{% endif %}); + + console.log("Response:", response.text); + } catch (error) { + console.error("Error:", error.message); + } +{% endfor %} +{% endblock %} + +{% block cleanup %} + // Flush and shutdown the Sentry exporter to ensure all spans are sent + const sentryExporter = mastra?.observability?.configs?.sentry?.exporters?.[0]; + if (sentryExporter && typeof sentryExporter.flush === 'function') { + await sentryExporter.flush(); + } + if (sentryExporter && typeof sentryExporter.shutdown === 'function') { + await sentryExporter.shutdown(); + } +{% endblock %} +} + +main() + .catch(console.error) + .finally(async () => { + // Give time for any pending network requests + await new Promise(resolve => setTimeout(resolve, 2000)); + }); diff --git a/src/runner/templates/agents/js/vercel/config.json b/src/runner/templates/agents/js/vercel/config.json new file mode 100644 index 0000000..75aaab9 --- /dev/null +++ b/src/runner/templates/agents/js/vercel/config.json @@ -0,0 +1,18 @@ +{ + "name": "vercel", + "displayName": "Vercel AI SDK", + "type": "agentic", + "platform": "js", + "dependencies": [ + { + "package": "ai", + "version": "framework" + }, + { + "package": "@ai-sdk/openai", + "version": "latest" + } + ], + "versions": ["4.3.16"], + "sentryVersions": ["10.28.0", "latest"] +} diff --git a/src/runner/templates/agents/js/vercel/template.njk b/src/runner/templates/agents/js/vercel/template.njk new file mode 100644 index 0000000..e273c66 --- /dev/null +++ b/src/runner/templates/agents/js/vercel/template.njk @@ -0,0 +1,79 @@ +{% extends "base.js.njk" %} + +{# Macro to check if content has images #} +{% macro hasImages(content) %} +{% if content is iterable and content is not string %}true{% else %}false{% endif %} +{% endmacro %} + +{# Macro to render user content for Vercel AI SDK #} +{% macro renderVercelContent(content) %} +{% if content is string %} +"{{ content }}" +{%- elif content is iterable %} +[ +{% for part in content %} +{% if part.type == 'text' %} + { type: "text", text: "{{ part.text }}" }, +{% elif part.type == 'image' %} + { type: "image", image: "data:{{ part.mediaType }};base64,{{ part.base64 }}" }, +{% endif %} +{% endfor %} + ] +{%- endif %} +{% endmacro %} + +{% block setup %} +// Functions will be initialized after dynamic import +let generateText, openai; +{% endblock %} + +{% block dynamic_imports %} + // Dynamic import of vercel ai AFTER Sentry.init() to ensure instrumentation + const ai = await import("ai"); + const aiSdkOpenai = await import("@ai-sdk/openai"); + generateText = ai.generateText; + openai = aiSdkOpenai.openai; +{% endblock %} + +{% block test %} +{% for input in inputs %} +{% set system_content = "" %} +{% set user_content = null %} +{% for message in input.messages %} +{% if message.role == 'system' %}{% set system_content = message.content %}{% endif %} +{% if message.role == 'user' %}{% set user_content = message.content %}{% endif %} +{% endfor %} + // Request {{ loop.index }}{% if loop.length > 1 %} of {{ loop.length }}{% endif %} + + try { +{% if user_content is iterable and user_content is not string %} + // Multimodal content - use messages array + const { text } = await generateText({ + model: openai({% if causeAPIError %}"invalid-model"{% else %}"{{ input.model }}"{% endif %}), +{% if system_content %} + system: "{{ system_content }}", +{% endif %} + messages: [ + { + role: "user", + content: {{ renderVercelContent(user_content) }}, + }, + ], + }); +{% else %} + // Simple text prompt + const { text } = await generateText({ + model: openai({% if causeAPIError %}"invalid-model"{% else %}"{{ input.model }}"{% endif %}), +{% if system_content %} + system: "{{ system_content }}", +{% endif %} + prompt: "{{ user_content }}", + }); +{% endif %} + console.log("Response:", text); + } catch (error) { + Sentry.captureException(error); + console.error("Error:", error.message); + } +{% endfor %} +{% endblock %} diff --git a/src/runner/templates/agents/py/google-genai/config.json b/src/runner/templates/agents/py/google-genai/config.json new file mode 100644 index 0000000..4e16a12 --- /dev/null +++ b/src/runner/templates/agents/py/google-genai/config.json @@ -0,0 +1,19 @@ +{ + "name": "google-genai", + "displayName": "Google GenAI Python SDK", + "type": "agentic", + "platform": "py", + "executionMode": "both", + "dependencies": [ + { + "package": "google-genai", + "version": "framework" + } + ], + "versions": ["1.61.0"], + "sentryVersions": ["2.51.0", "latest"], + "modelOverrides": { + "request": "gemini-2.0-flash", + "response": "gemini-2.0-flash*" + } +} diff --git a/src/runner/templates/agents/py/google-genai/template.njk b/src/runner/templates/agents/py/google-genai/template.njk new file mode 100644 index 0000000..75e3e31 --- /dev/null +++ b/src/runner/templates/agents/py/google-genai/template.njk @@ -0,0 +1,157 @@ +{% extends "base.py.njk" %} + +{% block imports %} +{{ super() }} +from google import genai +from google.genai.types import HttpOptions, Tool, FunctionDeclaration +{% endblock %} + +{% block setup %} +# Initialize Google GenAI client +client = genai.Client( + vertexai=True, + project=os.environ["GOOGLE_VERTEX_PROJECT"], + location=os.environ["GOOGLE_VERTEX_LOCATION"], + http_options=HttpOptions(api_version="v1"), +) + +{% if agent and agent.tools and agent.tools | length > 0 %} +# Define tools for the agent +{% for tool in agent.tools %} +def {{ tool.name }}({% for prop, details in tool.parameters.properties %}{{ prop }}: {{ 'float' if details.type == 'number' else details.type }}{% if not loop.last %}, {% endif %}{% endfor %}): + """{{ tool.description }}""" + return {{ tool.result }} + +{% endfor %} + +# Create function declarations for tools +tools = [ + Tool(function_declarations=[ +{% for tool in agent.tools %} + FunctionDeclaration( + name="{{ tool.name }}", + description="{{ tool.description }}", + parameters={ + "type": "OBJECT", + "properties": { +{% for prop, details in tool.parameters.properties %} + "{{ prop }}": { + "type": "{{ 'NUMBER' if details.type == 'number' else details.type | upper }}", + "description": "{{ details.description }}" + }, +{% endfor %} + }, + "required": {{ tool.parameters.required | dump }} + } + ), +{% endfor %} + ]) +] +{% endif %} +{% endblock %} + +{# Macro to render contents for Google GenAI - handles both string and multimodal array #} +{% macro renderGoogleContents(content) %} +{% if content is string %} +"{{ content }}" +{%- elif content is iterable %} +[ +{% for part in content %} +{% if part.type == 'text' %} + {"text": "{{ part.text }}"}, +{% elif part.type == 'image' %} + {"inline_data": {"mime_type": "{{ part.mediaType }}", "data": "{{ part.base64 }}"}}, +{% endif %} +{% endfor %} + ] +{%- endif %} +{% endmacro %} + +{% block test %} +{% for input in inputs %} + # Turn {{ loop.index }}: Generate content with tools + {# Extract system message if present #} + {% set system_content = "" %} + {% set user_content = null %} + {% for message in input.messages %} + {% if message.role == 'system' %} + {% set system_content = message.content %} + {% elif message.role == 'user' %} + {% set user_content = message.content %} + {% endif %} + {% endfor %} + +{% if isAsync %} + response = await client.aio.models.generate_content( +{% else %} + response = client.models.generate_content( +{% endif %} + model="{{ input.model }}", + contents={{ renderGoogleContents(user_content) }}, +{% if agent and agent.tools and agent.tools | length > 0 %} + config=genai.types.GenerateContentConfig( +{% if system_content %} + system_instruction="{{ system_content }}", +{% endif %} + tools=tools, + ), +{% elif system_content %} + config=genai.types.GenerateContentConfig( + system_instruction="{{ system_content }}", + ), +{% endif %} + ) + + # Check if there's a function call in the response + function_call = None + if response.candidates and response.candidates[0].content.parts: + for part in response.candidates[0].content.parts: + if hasattr(part, 'function_call') and part.function_call: + function_call = part.function_call + break + + if function_call: + # Execute the function locally + func_name = function_call.name + func_args = dict(function_call.args) if function_call.args else {} + + print(f"Turn {{ loop.index }} Tool Call: {func_name}({func_args})") + + # Call the function (defined at module level) + result = globals()[func_name](**func_args) + print(f"Turn {{ loop.index }} Tool Result: {result}") + + # Send the function result back to the model +{% if isAsync %} + final_response = await client.aio.models.generate_content( +{% else %} + final_response = client.models.generate_content( +{% endif %} + model="{{ input.model }}", + contents=[ + {"role": "user", "parts": [{"text": "{{ user_content }}"}]}, + {"role": "model", "parts": [{"function_call": {"name": func_name, "args": func_args}}]}, + {"role": "user", "parts": [{"function_response": {"name": func_name, "response": {"result": result}}}]}, + ], +{% if agent and agent.tools and agent.tools | length > 0 %} + config=genai.types.GenerateContentConfig( +{% if system_content %} + system_instruction="{{ system_content }}", +{% endif %} + tools=tools, + ), +{% elif system_content %} + config=genai.types.GenerateContentConfig( + system_instruction="{{ system_content }}", + ), +{% endif %} + ) + print(f"Turn {{ loop.index }} Response: {final_response.text}") + else: + # Direct text response + print(f"Turn {{ loop.index }} Response: {response.text}") +{% if not loop.last %} + print() # Blank line between turns +{% endif %} +{% endfor %} +{% endblock %} diff --git a/src/runner/templates/agents/py/langgraph/config.json b/src/runner/templates/agents/py/langgraph/config.json new file mode 100644 index 0000000..c1bc2ea --- /dev/null +++ b/src/runner/templates/agents/py/langgraph/config.json @@ -0,0 +1,19 @@ +{ + "name": "langgraph", + "displayName": "LangGraph Python SDK", + "type": "agentic", + "platform": "py", + "executionMode": "both", + "dependencies": [ + { + "package": "langgraph", + "version": "framework" + }, + { + "package": "langchain-openai", + "version": "latest" + } + ], + "versions": ["1.0.7"], + "sentryVersions": ["2.51.0", "latest"] +} diff --git a/src/runner/templates/agents/py/langgraph/template.njk b/src/runner/templates/agents/py/langgraph/template.njk new file mode 100644 index 0000000..80951de --- /dev/null +++ b/src/runner/templates/agents/py/langgraph/template.njk @@ -0,0 +1,83 @@ +{% extends "base.py.njk" %} + +{# Macro to render message content for LangGraph (LangChain format) #} +{% macro renderLangGraphContent(content) %} +{% if content is string %} +"{{ content }}" +{%- elif content is iterable %} +[ +{% for part in content %} +{% if part.type == 'text' %} + {"type": "text", "text": "{{ part.text }}"}, +{% elif part.type == 'image' %} + {"type": "image_url", "image_url": {"url": "data:{{ part.mediaType }};base64,{{ part.base64 }}"}}, +{% endif %} +{% endfor %} + ] +{%- endif %} +{% endmacro %} + +{% block imports %} +{{ super() }} +from langgraph.prebuilt import create_react_agent +from langchain_openai import ChatOpenAI +from langchain_core.messages import HumanMessage, SystemMessage +{% endblock %} + +{% block setup %} +# Initialize ChatOpenAI for LangGraph +llm = ChatOpenAI( + model="{{ inputs[0].model }}", + api_key=os.environ.get("OPENAI_API_KEY"), +) + +{% if agent and agent.tools and agent.tools | length > 0 %} +# Define tools for the agent +{% for tool in agent.tools %} +def {{ tool.name }}({% for param in tool.parameters.required %}{{ param }}{% if not loop.last %}, {% endif %}{% endfor %}): + """{{ tool.description }}""" + {% if tool.result is defined %} + return {{ tool.result | tojson }} + {% else %} + return None + {% endif %} + +{% endfor %} +tools = [{% for tool in agent.tools %}{{ tool.name }}{% if not loop.last %}, {% endif %}{% endfor %}] +{% else %} +tools = [] +{% endif %} + +# Create react agent +agent = create_react_agent(llm, tools=tools) +{% endblock %} + +{% block test %} +{% for input in inputs %} + # Turn {{ loop.index }}: Run agent + messages = [ +{% for message in input.messages %} +{% if message.role == 'system' %} + SystemMessage(content="{{ message.content }}"), +{% elif message.role == 'user' %} + HumanMessage(content={{ renderLangGraphContent(message.content) }}), +{% endif %} +{% endfor %} + ] + +{% if isAsync %} + result = await agent.ainvoke({"messages": messages}) +{% else %} + result = agent.invoke({"messages": messages}) +{% endif %} + + # Extract the AI's response from the result + response_text = result["messages"][-1].content + + # Print response + print(f"Turn {{ loop.index }} Response: {response_text}") +{% if not loop.last %} + print() # Blank line between turns +{% endif %} +{% endfor %} +{% endblock %} diff --git a/src/runner/templates/agents/py/openai-agents/config.json b/src/runner/templates/agents/py/openai-agents/config.json new file mode 100644 index 0000000..ec09674 --- /dev/null +++ b/src/runner/templates/agents/py/openai-agents/config.json @@ -0,0 +1,15 @@ +{ + "name": "openai-agents", + "displayName": "OpenAI Agents SDK", + "type": "agentic", + "platform": "py", + "executionMode": "async", + "dependencies": [ + { + "package": "openai-agents", + "version": "framework" + } + ], + "versions": ["0.7.0"], + "sentryVersions": ["2.51.0", "latest"] +} diff --git a/src/runner/templates/agents/py/openai-agents/template.njk b/src/runner/templates/agents/py/openai-agents/template.njk new file mode 100644 index 0000000..507331d --- /dev/null +++ b/src/runner/templates/agents/py/openai-agents/template.njk @@ -0,0 +1,66 @@ +{% extends "base.py.njk" %} + +{# Macro to render multimodal content for OpenAI Agents SDK #} +{# For simple text, pass as string. For multimodal, use message format with content array #} +{% macro renderMultimodalContent(content) -%} +{% if content is string -%} +"{{ content }}" +{%- else -%} +[{"role": "user", "content": [{% for part in content %}{% if part.type == 'text' %}{"type": "input_text", "text": "{{ part.text }}"}{% elif part.type == 'image' %}{"type": "input_image", "image_url": "data:{{ part.mediaType }};base64,{{ part.base64 }}"}{% endif %}{% if not loop.last %}, {% endif %}{% endfor %}]}] +{%- endif -%} +{%- endmacro %} + +{% block imports %} +{{ super() }} +from agents import function_tool, Agent, Runner +{% endblock %} + +{% block setup %} +{% if agent.tools | length > 0 %} +# Define tool functions using @function_tool decorator +{% for tool in agent.tools %} +@function_tool +def {{ tool.name }}({% for param in tool.parameters.required %}{{ param }}: str{% if not loop.last %}, {% endif %}{% endfor %}): + """{{ tool.description }}""" + {% if tool.result is defined %} + return {{ tool.result | tojson }} + {% elif tool.error is defined %} + raise Exception("{{ tool.error }}") + {% else %} + return None + {% endif %} + +{% endfor %} +{% endif %} +# Create agent with tools +agent = Agent( + name="{{ agent.name }}", + instructions="{{ agent.description }}", + tools=[{% for tool in agent.tools %}{{ tool.name }}{% if not loop.last %}, {% endif %}{% endfor %}], + model="{{ inputs[0].model }}", +) +{% endblock %} + +{% block test %} +{% for input in inputs %} + # Turn {{ loop.index }}: Build prompt from messages +{% set user_content = null %} +{% for message in input.messages %} +{% if message.role == 'user' %}{% set user_content = message.content %}{% endif %} +{% endfor %} + prompt = {{ renderMultimodalContent(user_content) }} + + # Run agent +{% if isAsync %} + result = await Runner.run(agent, input=prompt) +{% else %} + result = Runner.run_sync(agent, input=prompt) +{% endif %} + + # Print result + print(f"Turn {{ loop.index }} Agent result: {result.final_output}") +{% if not loop.last %} + print() # Blank line between turns +{% endif %} +{% endfor %} +{% endblock %} diff --git a/src/runner/templates/agents/py/pydantic-ai/config.json b/src/runner/templates/agents/py/pydantic-ai/config.json new file mode 100644 index 0000000..bdcb879 --- /dev/null +++ b/src/runner/templates/agents/py/pydantic-ai/config.json @@ -0,0 +1,15 @@ +{ + "name": "pydantic-ai", + "displayName": "Pydantic AI Python SDK", + "type": "agentic", + "platform": "py", + "executionMode": "async", + "dependencies": [ + { + "package": "pydantic-ai", + "version": "framework" + } + ], + "versions": ["1.52.0"], + "sentryVersions": ["2.51.0", "latest"] +} diff --git a/src/runner/templates/agents/py/pydantic-ai/template.njk b/src/runner/templates/agents/py/pydantic-ai/template.njk new file mode 100644 index 0000000..a32dad2 --- /dev/null +++ b/src/runner/templates/agents/py/pydantic-ai/template.njk @@ -0,0 +1,73 @@ +{% extends "base.py.njk" %} + +{% block imports %} +{{ super() }} +from pydantic_ai import Agent, ImageUrl, RunContext +{% endblock %} + +{% block setup %} +{# Extract system message from first input #} +{% set system_content = "" %} +{% for message in inputs[0].messages %} + {% if message.role == 'system' %} + {% set system_content = message.content %} + {% endif %} +{% endfor %} + +# Create Pydantic AI agent with system prompt +agent = Agent( + "openai:{{ inputs[0].model }}", + system_prompt="{{ system_content }}", +) + +{% if agent.tools | length > 0 %} +# Register tools with the agent +{% for tool in agent.tools %} +@agent.tool +def {{ tool.name }}(context: RunContext, {% for param in tool.parameters.required %}{{ param }}: {% if tool.parameters.properties[param].type == 'number' %}float{% else %}str{% endif %}{% if not loop.last %}, {% endif %}{% endfor %}): + """{{ tool.description }}""" + {% if tool.error is defined %} + raise Exception("{{ tool.error }}") + {% elif tool.result is defined %} + return {{ tool.result | tojson }} + {% else %} + return None + {% endif %} + +{% endfor %} +{% endif %} +{% endblock %} + +{% block test %} +{% for input in inputs %} + # Turn {{ loop.index }}: Run agent + {# Extract user message content #} + {% set user_content = null %} + {% for message in input.messages %} + {% if message.role == 'user' %} + {% set user_content = message.content %} + {% endif %} + {% endfor %} + +{% if user_content is string %} + result = await agent.run("{{ user_content }}") +{% else %} + # Multimodal content - build message parts + message_parts = [] +{% for part in user_content %} +{% if part.type == 'text' %} + message_parts.append("{{ part.text }}") +{% elif part.type == 'image' %} + message_parts.append(ImageUrl(url="data:{{ part.mediaType }};base64,{{ part.base64 }}")) +{% endif %} +{% endfor %} + result = await agent.run(message_parts) +{% endif %} + + # Print response + print(f"Turn {{ loop.index }} Response: {result.output}") +{% if not loop.last %} + print() # Blank line between turns +{% endif %} +{% endfor %} +{% endblock %} diff --git a/src/runner/templates/base.js.njk b/src/runner/templates/base.js.njk new file mode 100644 index 0000000..fe10904 --- /dev/null +++ b/src/runner/templates/base.js.njk @@ -0,0 +1,33 @@ +/** + * Sentry AI SDK Integration Test: {{ testName }} + * Framework: {{ frameworkName }} + * + * This test verifies that Sentry correctly instruments {{ frameworkName }} SDK calls. + */ + +// Initialize Sentry FIRST, before any other imports +import * as Sentry from "@sentry/node"; + +Sentry.init({ + dsn: process.env.SENTRY_DSN, + sendDefaultPii: true, + // Capture 100% of traces for testing + tracesSampleRate: 1.0, +}); + +{% block setup %}{% endblock %} + +async function main() { + await Sentry.startSpan( + { name: "{{ testName | lower | replace(' ', '-') }}", op: "test" }, + async () => { + // Dynamic imports AFTER Sentry.init() to ensure instrumentation hooks are in place +{% block dynamic_imports %}{% endblock %} +{% block test %}{% endblock %} + } + ); +} + +main() + .catch(console.error) + .finally(() => Sentry.flush(5000)); diff --git a/src/runner/templates/base.py.njk b/src/runner/templates/base.py.njk new file mode 100644 index 0000000..ecf35f1 --- /dev/null +++ b/src/runner/templates/base.py.njk @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +""" +Generated test file for {{ testName }} +Framework: {{ frameworkName }} +Execution mode: {% if isAsync %}async{% else %}sync{% endif %}{% if isStreaming is defined %} + +Streaming: {% if isStreaming %}yes{% else %}no{% endif %}{% endif %} +""" + +{% block imports %} +import os +import sentry_sdk +{% if isAsync %} +import asyncio +{% endif %} +{% endblock %} + +{% block inject_api_error %} +{% endblock %} + +{% block sdk_setup %} +sentry_sdk.init( + dsn=os.environ.get("SENTRY_DSN"), + traces_sample_rate=1.0, + send_default_pii=True, +) +{% endblock %} + +{% block setup %} +{% endblock %} + +{% if isAsync %} +async def main(): +{% else %} +def main(): +{% endif %} +{% block test %} + pass +{% endblock %} + +if __name__ == "__main__": +{% if causeAPIError %} + try: + with sentry_sdk.start_transaction(op="test", name="{{ testName }}"): +{% if isAsync %} + asyncio.run(main()) +{% else %} + main() +{% endif %} + except Exception as e: + print(f"Expected error: {type(e).__name__}: {e}") + sentry_sdk.capture_exception(e) + finally: + sentry_sdk.flush(timeout=5) +{% else %} + with sentry_sdk.start_transaction(op="test", name="{{ testName }}"): +{% if isAsync %} + asyncio.run(main()) +{% else %} + main() +{% endif %} + sentry_sdk.flush(timeout=5) +{% endif %} diff --git a/src/runner/templates/llm/README.md b/src/runner/templates/llm/README.md new file mode 100644 index 0000000..4c53e57 --- /dev/null +++ b/src/runner/templates/llm/README.md @@ -0,0 +1,78 @@ +# LLM Framework Templates + +Templates for frameworks that support direct LLM calls (no agent wrapper). + +## Compatible Frameworks + +| Platform | Framework | Directory | Status | +|----------|-----------|-----------|--------| +| JavaScript | OpenAI SDK | `js/openai/` | 🚧 TODO | +| JavaScript | Anthropic SDK | `js/anthropic/` | 🚧 TODO | +| JavaScript | Google GenAI | `js/google-genai/` | 🚧 TODO | +| Python | OpenAI SDK | `py/openai/` | ✅ Done | +| Python | Anthropic SDK | `py/anthropic/` | 🚧 TODO | +| Python | Google GenAI | `py/google-genai/` | 🚧 TODO | +| Python | LiteLLM | `py/litellm/` | 🚧 TODO | + +## Test Compatibility + +LLM templates should implement tests that have: +- `system` property (system message) +- `input.model` property (model identifier) +- `input.prompt` property (user prompt) + +Example test from `test-cases/llm/basic.ts`: +```typescript +{ + system: 'You are a helpful assistant.', + input: { + model: 'gpt-4o', + prompt: 'What is the capital of France?', + } +} +``` + +## Template Requirements + +Each LLM template must: + +1. **Extend base template** + ```nunjucks + {% extends "base.{js,py}.njk" %} + ``` + +2. **Import SDK** + ```nunjucks + {% block imports %} + {{ super() }} + from framework import Client + {% endblock %} + ``` + +3. **Initialize client** + ```nunjucks + {% block setup %} + client = Client(api_key=os.environ.get("FRAMEWORK_API_KEY")) + {% endblock %} + ``` + +4. **Call LLM** + ```nunjucks + {% block test %} + response = client.chat.create( + model="{{ input.model }}", + messages=[ + {"role": "system", "content": "{{ system }}"}, + {"role": "user", "content": "{{ input.prompt }}"} + ] + ) + print(response.content) + {% endblock %} + ``` + +## Notes + +- Use `{{ system }}` for system message +- Use `{{ input.model }}` for model +- Use `{{ input.prompt }}` for user prompt +- Framework-specific parameters can be added as needed diff --git a/src/runner/templates/llm/js/anthropic/config.json b/src/runner/templates/llm/js/anthropic/config.json new file mode 100644 index 0000000..e477242 --- /dev/null +++ b/src/runner/templates/llm/js/anthropic/config.json @@ -0,0 +1,19 @@ +{ + "name": "anthropic", + "displayName": "Anthropic JavaScript SDK", + "type": "llm-only", + "platform": "js", + "streamingMode": "both", + "dependencies": [ + { + "package": "@anthropic-ai/sdk", + "version": "framework" + } + ], + "versions": ["0.39.0"], + "sentryVersions": ["10.28.0", "latest"], + "modelOverrides": { + "request": "claude-haiku-4-5", + "response": "claude-haiku-4-5*" + } +} diff --git a/src/runner/templates/llm/js/anthropic/template.njk b/src/runner/templates/llm/js/anthropic/template.njk new file mode 100644 index 0000000..463b97e --- /dev/null +++ b/src/runner/templates/llm/js/anthropic/template.njk @@ -0,0 +1,85 @@ +{% extends "base.js.njk" %} + +{# Macro to render message content for Anthropic - handles both string and multimodal array #} +{% macro renderContent(content) %} +{% if content is string %} +"{{ content }}" +{%- elif content is iterable %} +[ +{% for part in content %} +{% if part.type == 'text' %} + { type: "text", text: "{{ part.text }}" }, +{% elif part.type == 'image' %} + { type: "image", source: { type: "base64", media_type: "{{ part.mediaType }}", data: "{{ part.base64 }}" } }, +{% endif %} +{% endfor %} + ] +{%- endif %} +{% endmacro %} + +{% block setup %} +// Client will be initialized after dynamic import +let client; +{% endblock %} + +{% block dynamic_imports %} + // Dynamic import of anthropic AFTER Sentry.init() to ensure instrumentation + const Anthropic = (await import("@anthropic-ai/sdk")).default; + client = new Anthropic(); +{% endblock %} + +{% block test %} +{% for input in inputs %} +{% set system_content = "" %} +{% for message in input.messages %} +{% if message.role == 'system' %}{% set system_content = message.content %}{% endif %} +{% endfor %} + // Request {{ loop.index }}{% if loop.length > 1 %} of {{ loop.length }}{% endif %} + + try { +{% if isStreaming %} + const stream = await client.messages.stream({ + model: {% if causeAPIError %}"invalid-model"{% else %}"{{ input.model }}"{% endif %}, + max_tokens: 1024, +{% if system_content %} + system: "{{ system_content }}", +{% endif %} + messages: [ +{% for message in input.messages %} +{% if message.role != 'system' %} + { role: "{{ message.role }}", content: {{ renderContent(message.content) }} }, +{% endif %} +{% endfor %} + ], + }); + + const chunks = []; + for await (const event of stream) { + if (event.type === "content_block_delta" && event.delta.type === "text_delta") { + chunks.push(event.delta.text); + } + } + console.log("Response:", chunks.join("")); +{% else %} + const response = await client.messages.create({ + model: {% if causeAPIError %}"invalid-model"{% else %}"{{ input.model }}"{% endif %}, + max_tokens: 1024, +{% if system_content %} + system: "{{ system_content }}", +{% endif %} + messages: [ +{% for message in input.messages %} +{% if message.role != 'system' %} + { role: "{{ message.role }}", content: {{ renderContent(message.content) }} }, +{% endif %} +{% endfor %} + ], + }); + console.log("Response:", response.content[0].text); +{% endif %} + } catch (error) { + Sentry.captureException(error); + console.error("Error:", error.message); + } +{% endfor %} +{% endblock %} diff --git a/src/runner/templates/llm/js/google-genai/config.json b/src/runner/templates/llm/js/google-genai/config.json new file mode 100644 index 0000000..f0cf0db --- /dev/null +++ b/src/runner/templates/llm/js/google-genai/config.json @@ -0,0 +1,19 @@ +{ + "name": "google-genai", + "displayName": "Google GenAI JavaScript SDK", + "type": "llm-only", + "platform": "js", + "streamingMode": "both", + "dependencies": [ + { + "package": "@google/genai", + "version": "framework" + } + ], + "versions": ["1.38.0"], + "sentryVersions": ["10.38.0", "latest"], + "modelOverrides": { + "request": "gemini-2.0-flash", + "response": "gemini-2.0-flash*" + } +} diff --git a/src/runner/templates/llm/js/google-genai/template.njk b/src/runner/templates/llm/js/google-genai/template.njk new file mode 100644 index 0000000..00a6eb5 --- /dev/null +++ b/src/runner/templates/llm/js/google-genai/template.njk @@ -0,0 +1,77 @@ +{% extends "base.js.njk" %} + +{# Macro to render contents for Google GenAI - handles both string and multimodal array #} +{% macro renderContents(content) %} +{% if content is string %} +"{{ content }}" +{%- elif content is iterable %} +[ +{% for part in content %} +{% if part.type == 'text' %} + { text: "{{ part.text }}" }, +{% elif part.type == 'image' %} + { inlineData: { mimeType: "{{ part.mediaType }}", data: "{{ part.base64 }}" } }, +{% endif %} +{% endfor %} + ] +{%- endif %} +{% endmacro %} + +{% block setup %} +// Client will be initialized after dynamic import +let client; +{% endblock %} + +{% block dynamic_imports %} + // Dynamic import of google-genai AFTER Sentry.init() to ensure instrumentation + const { GoogleGenAI } = await import("@google/genai"); + client = new GoogleGenAI({ + vertexai: true, + project: process.env.GOOGLE_VERTEX_PROJECT, + location: process.env.GOOGLE_VERTEX_LOCATION, + }); +{% endblock %} + +{% block test %} +{% for input in inputs %} +{% set system_content = "" %} +{% set user_content = null %} +{% for message in input.messages %} +{% if message.role == 'system' %}{% set system_content = message.content %}{% endif %} +{% if message.role == 'user' %}{% set user_content = message.content %}{% endif %} +{% endfor %} + // Request {{ loop.index }}{% if loop.length > 1 %} of {{ loop.length }}{% endif %} + + try { +{% if isStreaming %} + const stream = await client.models.generateContentStream({ + model: {% if causeAPIError %}"invalid-model"{% else %}"{{ input.model }}"{% endif %}, + contents: {{ renderContents(user_content) }}, +{% if system_content %} + config: { systemInstruction: ["{{ system_content }}"] }, +{% endif %} + }); + + const chunks = []; + for await (const chunk of stream) { + if (chunk.text) { + chunks.push(chunk.text); + } + } + console.log("Response:", chunks.join("")); +{% else %} + const response = await client.models.generateContent({ + model: {% if causeAPIError %}"invalid-model"{% else %}"{{ input.model }}"{% endif %}, + contents: {{ renderContents(user_content) }}, +{% if system_content %} + config: { systemInstruction: ["{{ system_content }}"] }, +{% endif %} + }); + console.log("Response:", response.text); +{% endif %} + } catch (error) { + Sentry.captureException(error); + console.error("Error:", error.message); + } +{% endfor %} +{% endblock %} diff --git a/src/runner/templates/llm/js/langchain/config.json b/src/runner/templates/llm/js/langchain/config.json new file mode 100644 index 0000000..54ec2fc --- /dev/null +++ b/src/runner/templates/llm/js/langchain/config.json @@ -0,0 +1,23 @@ +{ + "name": "langchain", + "displayName": "LangChain JavaScript SDK", + "type": "llm-only", + "platform": "js", + "streamingMode": "both", + "dependencies": [ + { + "package": "langchain", + "version": "framework" + }, + { + "package": "@langchain/openai", + "version": "0.3.0" + }, + { + "package": "@langchain/core", + "version": "0.3.49" + } + ], + "versions": ["0.3.24"], + "sentryVersions": ["10.38.0", "latest"] +} diff --git a/src/runner/templates/llm/js/langchain/template.njk b/src/runner/templates/llm/js/langchain/template.njk new file mode 100644 index 0000000..e7a687a --- /dev/null +++ b/src/runner/templates/llm/js/langchain/template.njk @@ -0,0 +1,74 @@ +{% extends "base.js.njk" %} + +{# Macro to render message content for LangChain - handles both string and multimodal array #} +{% macro renderLangChainContent(content) %} +{% if content is string %} +"{{ content }}" +{%- elif content is iterable %} +[ +{% for part in content %} +{% if part.type == 'text' %} + { type: "text", text: "{{ part.text }}" }, +{% elif part.type == 'image' %} + { type: "image_url", image_url: { url: "data:{{ part.mediaType }};base64,{{ part.base64 }}" } }, +{% endif %} +{% endfor %} + ] +{%- endif %} +{% endmacro %} + +{% block setup %} +// Classes and client will be initialized after dynamic import +let ChatOpenAI, HumanMessage, SystemMessage, AIMessage; +let chat; +{% endblock %} + +{% block dynamic_imports %} + // Dynamic import of langchain AFTER Sentry.init() to ensure instrumentation + const langchainOpenAI = await import("@langchain/openai"); + const langchainMessages = await import("@langchain/core/messages"); + ChatOpenAI = langchainOpenAI.ChatOpenAI; + HumanMessage = langchainMessages.HumanMessage; + SystemMessage = langchainMessages.SystemMessage; + AIMessage = langchainMessages.AIMessage; + chat = new ChatOpenAI({ + modelName: {% if causeAPIError %}"invalid-model"{% else %}"{{ inputs[0].model }}"{% endif %}, + }); +{% endblock %} + +{% block test %} +{% for input in inputs %} + // Request {{ loop.index }}{% if loop.length > 1 %} of {{ loop.length }}{% endif %} + + const messages{{ loop.index }} = [ +{% for message in input.messages %} +{% if message.role == 'system' %} + new SystemMessage("{{ message.content }}"), +{% elif message.role == 'user' %} + new HumanMessage({ content: {{ renderLangChainContent(message.content) }} }), +{% elif message.role == 'assistant' %} + new AIMessage("{{ message.content }}"), +{% endif %} +{% endfor %} + ]; + + try { +{% if isStreaming %} + const stream = await chat.stream(messages{{ loop.index }}); + const chunks = []; + for await (const chunk of stream) { + if (chunk.content) { + chunks.push(chunk.content); + } + } + console.log("Response:", chunks.join("")); +{% else %} + const response = await chat.invoke(messages{{ loop.index }}); + console.log("Response:", response.content); +{% endif %} + } catch (error) { + Sentry.captureException(error); + console.error("Error:", error.message); + } +{% endfor %} +{% endblock %} diff --git a/src/runner/templates/llm/js/openai/config.json b/src/runner/templates/llm/js/openai/config.json new file mode 100644 index 0000000..3621e08 --- /dev/null +++ b/src/runner/templates/llm/js/openai/config.json @@ -0,0 +1,15 @@ +{ + "name": "openai", + "displayName": "OpenAI JavaScript SDK", + "type": "llm-only", + "platform": "js", + "streamingMode": "both", + "dependencies": [ + { + "package": "openai", + "version": "framework" + } + ], + "versions": ["4.96.0"], + "sentryVersions": ["10.28.0", "latest"] +} diff --git a/src/runner/templates/llm/js/openai/template.njk b/src/runner/templates/llm/js/openai/template.njk new file mode 100644 index 0000000..91548db --- /dev/null +++ b/src/runner/templates/llm/js/openai/template.njk @@ -0,0 +1,71 @@ +{% extends "base.js.njk" %} + +{# Macro to render message content - handles both string and multimodal array #} +{% macro renderContent(content) %} +{% if content is string %} +"{{ content }}" +{%- elif content is iterable %} +[ +{% for part in content %} +{% if part.type == 'text' %} + { type: "text", text: "{{ part.text }}" }, +{% elif part.type == 'image' %} + { type: "image_url", image_url: { url: "data:{{ part.mediaType }};base64,{{ part.base64 }}" } }, +{% endif %} +{% endfor %} + ] +{%- endif %} +{% endmacro %} + +{% block setup %} +// Client will be initialized after dynamic import +let client; +{% endblock %} + +{% block dynamic_imports %} + // Dynamic import of openai AFTER Sentry.init() to ensure instrumentation + const OpenAI = (await import("openai")).default; + client = new OpenAI(); +{% endblock %} + +{% block test %} +{% for input in inputs %} + // Request {{ loop.index }}{% if loop.length > 1 %} of {{ loop.length }}{% endif %} + + try { +{% if isStreaming %} + const stream = await client.chat.completions.create({ + model: {% if causeAPIError %}"invalid-model"{% else %}"{{ input.model }}"{% endif %}, + messages: [ +{% for message in input.messages %} + { role: "{{ message.role }}", content: {{ renderContent(message.content) }} }, +{% endfor %} + ], + stream: true, + }); + + const chunks = []; + for await (const chunk of stream) { + const content = chunk.choices[0]?.delta?.content; + if (content) { + chunks.push(content); + } + } + console.log("Response:", chunks.join("")); +{% else %} + const response = await client.chat.completions.create({ + model: {% if causeAPIError %}"invalid-model"{% else %}"{{ input.model }}"{% endif %}, + messages: [ +{% for message in input.messages %} + { role: "{{ message.role }}", content: {{ renderContent(message.content) }} }, +{% endfor %} + ], + }); + console.log("Response:", response.choices[0].message.content); +{% endif %} + } catch (error) { + Sentry.captureException(error); + console.error("Error:", error.message); + } +{% endfor %} +{% endblock %} diff --git a/src/runner/templates/llm/py/anthropic/config.json b/src/runner/templates/llm/py/anthropic/config.json new file mode 100644 index 0000000..92f21af --- /dev/null +++ b/src/runner/templates/llm/py/anthropic/config.json @@ -0,0 +1,24 @@ +{ + "name": "anthropic", + "displayName": "Anthropic Python SDK", + "type": "llm-only", + "platform": "py", + "executionMode": "both", + "streamingMode": "both", + "dependencies": [ + { + "package": "anthropic", + "version": "framework" + }, + { + "package": "respx", + "version": "latest" + } + ], + "versions": ["0.77.0"], + "sentryVersions": ["2.51.0", "latest"], + "modelOverrides": { + "request": "claude-haiku-4-5", + "response": "claude-haiku-4-5*" + } +} diff --git a/src/runner/templates/llm/py/anthropic/template.njk b/src/runner/templates/llm/py/anthropic/template.njk new file mode 100644 index 0000000..34bd3df --- /dev/null +++ b/src/runner/templates/llm/py/anthropic/template.njk @@ -0,0 +1,101 @@ +{% extends "base.py.njk" %} + +{# Macro to convert content to Anthropic format #} +{% macro convertContent(content) %} +{% if content is string %} +"{{ content }}" +{%- elif content is iterable %} +[ +{% for part in content %} +{% if part.type == 'text' %} + {"type": "text", "text": "{{ part.text }}"}, +{% elif part.type == 'image' %} + {"type": "image", "source": {"type": "base64", "media_type": "{{ part.mediaType }}", "data": "{{ part.base64 }}"}}, +{% endif %} +{% endfor %} + ] +{%- endif %} +{% endmacro %} + +{% block imports %} +{{ super() }} +{% if isAsync %} +from anthropic import AsyncAnthropic +{% else %} +from anthropic import Anthropic +{% endif %} +{% if causeAPIError %} +import respx +import httpx +{% endif %} +{% endblock %} + +{% block inject_api_error %} +{% if causeAPIError %} +respx.route(host="api.anthropic.com").mock(return_value=httpx.Response(500, json={"error": {"message": "Server error", "type": "server_error"}})) +respx.mock.start() +{% endif %} +{% endblock %} + +{% block setup %} +{% if isAsync %} +client = AsyncAnthropic() +{% else %} +client = Anthropic() +{% endif %} +{% endblock %} + +{% block test %} +{% for input in inputs %} + # Turn {{ loop.index }} + system_prompt = None + messages = [] +{% for message in input.messages %} +{% if message.role == 'system' %} + system_prompt = "{{ message.content }}" +{% else %} + messages.append({"role": "{{ message.role }}", "content": {{ convertContent(message.content) }}}) +{% endif %} +{% endfor %} + + kwargs = { + "model": "{{ input.model }}", + "max_tokens": 1024, + "messages": messages, + } + if system_prompt: + kwargs["system"] = system_prompt + +{% if isStreaming %} +{% if isAsync %} + async with client.messages.stream(**kwargs) as stream: + collected_content = [] + async for event in stream: + if event.type == "content_block_delta": + collected_content.append(event.delta.text) + + full_response = "".join(collected_content) + print(f"Turn {{ loop.index }} Response: {full_response}") +{% else %} + with client.messages.stream(**kwargs) as stream: + collected_content = [] + for event in stream: + if event.type == "content_block_delta": + collected_content.append(event.delta.text) + + full_response = "".join(collected_content) + print(f"Turn {{ loop.index }} Response: {full_response}") +{% endif %} +{% else %} +{% if isAsync %} + response = await client.messages.create(**kwargs) +{% else %} + response = client.messages.create(**kwargs) +{% endif %} + print(f"Turn {{ loop.index }} Response: {response.content[0].text}") +{% endif %} +{% if not loop.last %} + +{% endif %} +{% endfor %} +{% endblock %} diff --git a/src/runner/templates/llm/py/langchain/config.json b/src/runner/templates/llm/py/langchain/config.json new file mode 100644 index 0000000..15fdd8c --- /dev/null +++ b/src/runner/templates/llm/py/langchain/config.json @@ -0,0 +1,24 @@ +{ + "name": "langchain", + "displayName": "LangChain Python SDK", + "type": "llm-only", + "platform": "py", + "executionMode": "both", + "streamingMode": "both", + "dependencies": [ + { + "package": "langchain", + "version": "framework" + }, + { + "package": "langchain-openai", + "version": "latest" + }, + { + "package": "respx", + "version": "latest" + } + ], + "versions": ["1.2.8"], + "sentryVersions": ["2.51.0", "latest"] +} diff --git a/src/runner/templates/llm/py/langchain/template.njk b/src/runner/templates/llm/py/langchain/template.njk new file mode 100644 index 0000000..32e3d25 --- /dev/null +++ b/src/runner/templates/llm/py/langchain/template.njk @@ -0,0 +1,81 @@ +{% extends "base.py.njk" %} + +{# Macro to convert content to LangChain format (uses OpenAI-style for images) #} +{% macro convertLangChainContent(content) %} +{% if content is string %} +"{{ content }}" +{%- elif content is iterable %} +[ +{% for part in content %} +{% if part.type == 'text' %} + {"type": "text", "text": "{{ part.text }}"}, +{% elif part.type == 'image' %} + {"type": "image_url", "image_url": {"url": "data:{{ part.mediaType }};base64,{{ part.base64 }}"}}, +{% endif %} +{% endfor %} + ] +{%- endif %} +{% endmacro %} + +{% block imports %} +{{ super() }} +from langchain_openai import ChatOpenAI +from langchain_core.messages import HumanMessage, SystemMessage, AIMessage +{% if causeAPIError %} +import respx +import httpx +{% endif %} +{% endblock %} + +{% block inject_api_error %} +{% if causeAPIError %} +respx.route(host="api.openai.com").mock(return_value=httpx.Response(500, json={"error": {"message": "Server error", "type": "server_error"}})) +respx.mock.start() +{% endif %} +{% endblock %} + +{% block setup %} +{% set input = inputs[0] %} +chat = ChatOpenAI(model="{{ input.model }}") +{% endblock %} + +{% block test %} +{% for input in inputs %} + # Turn {{ loop.index }} + messages = [ +{% for message in input.messages %} + {% if message.role == 'system' %} + SystemMessage(content="{{ message.content }}"), + {% elif message.role == 'user' %} + HumanMessage(content={{ convertLangChainContent(message.content) }}), + {% elif message.role == 'assistant' %} + AIMessage(content="{{ message.content }}"), + {% endif %} +{% endfor %} + ] +{% if isStreaming %} +{% if isAsync %} + chunks = [] + async for chunk in chat.astream(messages): + chunks.append(chunk.content) + response_content = "".join(chunks) +{% else %} + chunks = [] + for chunk in chat.stream(messages): + chunks.append(chunk.content) + response_content = "".join(chunks) +{% endif %} + print(f"Turn {{ loop.index }} Response: {response_content}") +{% else %} +{% if isAsync %} + response = await chat.ainvoke(messages) +{% else %} + response = chat.invoke(messages) +{% endif %} + print(f"Turn {{ loop.index }} Response: {response.content}") +{% endif %} +{% if not loop.last %} + +{% endif %} +{% endfor %} +{% endblock %} diff --git a/src/runner/templates/llm/py/litellm/config.json b/src/runner/templates/llm/py/litellm/config.json new file mode 100644 index 0000000..01e9ffe --- /dev/null +++ b/src/runner/templates/llm/py/litellm/config.json @@ -0,0 +1,20 @@ +{ + "name": "litellm", + "displayName": "LiteLLM Python SDK", + "type": "llm-only", + "platform": "py", + "executionMode": "both", + "streamingMode": "both", + "dependencies": [ + { + "package": "litellm", + "version": "framework" + }, + { + "package": "respx", + "version": "latest" + } + ], + "versions": ["1.81.6"], + "sentryVersions": ["2.51.0", "latest"] +} diff --git a/src/runner/templates/llm/py/litellm/template.njk b/src/runner/templates/llm/py/litellm/template.njk new file mode 100644 index 0000000..18325ca --- /dev/null +++ b/src/runner/templates/llm/py/litellm/template.njk @@ -0,0 +1,98 @@ +{% extends "base.py.njk" %} + +{# Macro to convert our content format to OpenAI format (LiteLLM uses OpenAI format) #} +{% macro convertMessages(messages) %} +[ +{% for message in messages %} + { + "role": "{{ message.role }}", +{% if message.content is string %} + "content": "{{ message.content }}" +{% else %} + "content": [ +{% for part in message.content %} +{% if part.type == 'text' %} + {"type": "text", "text": "{{ part.text }}"}, +{% elif part.type == 'image' %} + {"type": "image_url", "image_url": {"url": "data:{{ part.mediaType }};base64,{{ part.base64 }}"}}, +{% endif %} +{% endfor %} + ] +{% endif %} + }, +{% endfor %} + ] +{% endmacro %} + +{% block imports %} +{{ super() }} +import time + +import litellm +from sentry_sdk.integrations.litellm import LiteLLMIntegration +from sentry_sdk.integrations.openai import OpenAIIntegration +{% if causeAPIError %} +import respx +import httpx +{% endif %} +{% endblock %} + +{% block inject_api_error %} +{% if causeAPIError %} +respx.route(host="api.openai.com").mock(return_value=httpx.Response(500, json={"error": {"message": "Server error", "type": "server_error"}})) +respx.mock.start() +{% endif %} +{% endblock %} + +{% block sdk_setup %} +sentry_sdk.init( + dsn=os.environ.get("SENTRY_DSN"), + traces_sample_rate=1.0, + send_default_pii=True, + integrations=[LiteLLMIntegration(include_prompts=True)], + disabled_integrations=[OpenAIIntegration()], +) +{% endblock %} + +{% block test %} +{% for input in inputs %} + # Turn {{ loop.index }} +{% if isStreaming %} +{% if isAsync %} + stream = await litellm.acompletion( +{% else %} + stream = litellm.completion( +{% endif %} + model="openai/{{ input.model }}", + messages={{ convertMessages(input.messages) }}, + stream=True, + ) + + collected_content = [] +{% if isAsync %} + async for chunk in stream: +{% else %} + for chunk in stream: +{% endif %} + if chunk.choices[0].delta.content is not None: + collected_content.append(chunk.choices[0].delta.content) + + full_response = "".join(collected_content) + print(f"Turn {{ loop.index }} Response: {full_response}") +{% else %} +{% if isAsync %} + response = await litellm.acompletion( +{% else %} + response = litellm.completion( +{% endif %} + model="openai/{{ input.model }}", + messages={{ convertMessages(input.messages) }}, + ) + print(f"Turn {{ loop.index }} Response: {response.choices[0].message.content}") +{% endif %} + time.sleep(0.1) # sleep is necessary for LiteLLM because it needs threaded callbacks to finish +{% if not loop.last %} + +{% endif %} +{% endfor %} +{% endblock %} diff --git a/src/runner/templates/llm/py/openai/config.json b/src/runner/templates/llm/py/openai/config.json new file mode 100644 index 0000000..5f940e6 --- /dev/null +++ b/src/runner/templates/llm/py/openai/config.json @@ -0,0 +1,23 @@ +{ + "name": "openai", + "displayName": "OpenAI Python SDK", + "type": "llm-only", + "platform": "py", + "executionMode": "both", + "streamingMode": "both", + "dependencies": [ + { + "package": "openai", + "version": "framework" + }, + { + "package": "respx", + "version": "latest" + } + ], + "versions": ["2.16.0"], + "sentryVersions": ["2.51.0", "latest"], + "matrix": { + "modelProviders": ["openai"] + } +} diff --git a/src/runner/templates/llm/py/openai/template.njk b/src/runner/templates/llm/py/openai/template.njk new file mode 100644 index 0000000..1fbb5ee --- /dev/null +++ b/src/runner/templates/llm/py/openai/template.njk @@ -0,0 +1,95 @@ +{% extends "base.py.njk" %} + +{# Macro to convert our content format to OpenAI format for Python #} +{% macro convertMessages(messages) %} +[ +{% for message in messages %} + { + "role": "{{ message.role }}", +{% if message.content is string %} + "content": "{{ message.content }}" +{% else %} + "content": [ +{% for part in message.content %} +{% if part.type == 'text' %} + {"type": "text", "text": "{{ part.text }}"}, +{% elif part.type == 'image' %} + {"type": "image_url", "image_url": {"url": "data:{{ part.mediaType }};base64,{{ part.base64 }}"}}, +{% endif %} +{% endfor %} + ] +{% endif %} + }, +{% endfor %} + ] +{% endmacro %} + +{% block imports %} +{{ super() }} +{% if isAsync %} +from openai import AsyncOpenAI +{% else %} +from openai import OpenAI +{% endif %} +{% if causeAPIError %} +import respx +import httpx +{% endif %} +{% endblock %} + +{% block inject_api_error %} +{% if causeAPIError %} +respx.route(host="api.openai.com").mock(return_value=httpx.Response(500, json={"error": {"message": "Server error", "type": "server_error"}})) +respx.mock.start() +{% endif %} +{% endblock %} + +{% block setup %} +{% if isAsync %} +client = AsyncOpenAI() +{% else %} +client = OpenAI() +{% endif %} +{% endblock %} + +{% block test %} +{% for input in inputs %} + # Turn {{ loop.index }} +{% if isStreaming %} +{% if isAsync %} + stream = await client.chat.completions.create( +{% else %} + stream = client.chat.completions.create( +{% endif %} + model="{{ input.model }}", + messages={{ convertMessages(input.messages) }}, + stream=True, + ) + + collected_content = [] +{% if isAsync %} + async for chunk in stream: +{% else %} + for chunk in stream: +{% endif %} + if chunk.choices[0].delta.content is not None: + collected_content.append(chunk.choices[0].delta.content) + + full_response = "".join(collected_content) + print(f"Turn {{ loop.index }} Response: {full_response}") +{% else %} +{% if isAsync %} + response = await client.chat.completions.create( +{% else %} + response = client.chat.completions.create( +{% endif %} + model="{{ input.model }}", + messages={{ convertMessages(input.messages) }}, + ) + print(f"Turn {{ loop.index }} Response: {response.choices[0].message.content}") +{% endif %} +{% if not loop.last %} + +{% endif %} +{% endfor %} +{% endblock %} diff --git a/src/setup.ts b/src/setup.ts new file mode 100644 index 0000000..9f2c3c0 --- /dev/null +++ b/src/setup.ts @@ -0,0 +1,227 @@ +#!/usr/bin/env node +/** + * CLI tool to setup test environments and render templates without executing tests. + * This creates the same structure as actual tests in runs/ directory. + */ + +import "dotenv/config"; +import { Orchestrator } from "./orchestrator.js"; +import { FrameworkConfig } from "./types.js"; +import { discoverFrameworks } from "./runner/framework-discovery.js"; +import { getAllTests } from "./test-cases/index.js"; + +interface SetupOptions { + platform?: "js" | "py"; + framework?: string; + test?: string; + sync?: boolean; + async?: boolean; + streaming?: boolean; + blocking?: boolean; + sentryPythonPath?: string; + sentryJavaScriptPath?: string; + verbose?: boolean; +} + +function parseArgs(): SetupOptions { + const args = process.argv.slice(2); + const options: SetupOptions = {}; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + const value = args[i + 1]; + + switch (arg) { + case "--platform": + case "-p": + if (value !== "js" && value !== "py") { + console.error('Error: --platform must be "js" or "py"'); + process.exit(1); + } + options.platform = value; + i++; + break; + case "--framework": + case "-f": + options.framework = value; + i++; + break; + case "--test": + case "-t": + options.test = value; + i++; + break; + case "--sync": + options.sync = true; + break; + case "--async": + options.async = true; + break; + case "--streaming": + options.streaming = true; + break; + case "--blocking": + options.blocking = true; + break; + case "--sentry-python": + options.sentryPythonPath = value; + i++; + break; + case "--sentry-javascript": + options.sentryJavaScriptPath = value; + i++; + break; + case "--verbose": + case "-v": + options.verbose = true; + break; + case "--help": + case "-h": + printHelp(); + process.exit(0); + default: + console.error(`Unknown option: ${arg}`); + printHelp(); + process.exit(1); + } + } + + return options; +} + +function printHelp(): void { + console.log(` +Setup - Create test environments and render templates without executing tests + +Usage: + npm run setup [options] + +Options: + -p, --platform Only setup for specific platform + -f, --framework Only setup for specific framework + -t, --test Only setup for specific test + --sync Only setup sync tests (Python only) + --async Only setup async tests (Python only) + --streaming Only setup streaming tests + --blocking Only setup blocking (non-streaming) tests + --sentry-python Use local Sentry Python SDK (editable install) + --sentry-javascript Use local Sentry JavaScript SDK (link) + -v, --verbose Show detailed output + -h, --help Show this help + +Examples: + npm run setup + npm run setup -- --platform py + npm run setup -- --framework openai + npm run setup -- --test "Basic LLM Test" + npm run setup -- --platform py --framework openai --sync --streaming + npm run setup -- --sentry-python ~/sentry-python + +Output: + Creates test environments in runs/ directory with: + - Virtual environments (Python) or node_modules (JavaScript) + - Installed dependencies + - Rendered test files + + Use 'npm start run' to execute the tests after setup. + `); +} + +async function main(): Promise { + const options = parseArgs(); + + console.log("Sentry AI SDK Integration Tests - Setup\n"); + + // Create orchestrator with setup-specific options (no span collector needed) + const orchestrator = new Orchestrator({ + verbose: options.verbose, + sync: options.sync, + async: options.async, + streaming: options.streaming, + blocking: options.blocking, + }); + + // Discover frameworks + let discoveredFrameworks = discoverFrameworks(); + + // Apply filters + if (options.platform) { + discoveredFrameworks = discoveredFrameworks.filter( + (f) => f.platform === options.platform, + ); + } + if (options.framework) { + discoveredFrameworks = discoveredFrameworks.filter( + (f) => f.name === options.framework, + ); + } + + if (discoveredFrameworks.length === 0) { + console.log("No frameworks found matching criteria."); + process.exit(1); + } + + // Load test definitions + let testDefinitions = getAllTests(); + if (options.test) { + testDefinitions = testDefinitions.filter((t) => t.name === options.test); + } + + if (testDefinitions.length === 0) { + console.log("No tests found matching criteria."); + process.exit(1); + } + + // Set local Sentry SDK paths if provided + if (options.sentryPythonPath) { + process.env.SENTRY_PYTHON_PATH = options.sentryPythonPath; + console.log(`Using local Sentry Python SDK: ${options.sentryPythonPath}\n`); + } + if (options.sentryJavaScriptPath) { + process.env.SENTRY_JAVASCRIPT_PATH = options.sentryJavaScriptPath; + console.log( + `Using local Sentry JavaScript SDK: ${options.sentryJavaScriptPath}\n`, + ); + } + + // Convert discovered frameworks to test matrix + const frameworks: FrameworkConfig[] = discoveredFrameworks.map((df) => { + // Determine Sentry version based on platform and local SDK paths + let sentryVersion = df.sentryVersions[0]; + if (df.platform === "py" && options.sentryPythonPath) { + sentryVersion = "local"; + } else if (df.platform === "js" && options.sentryJavaScriptPath) { + sentryVersion = "local"; + } + + return { + name: df.name, + platform: df.platform, + type: df.type, + version: df.versions[0], + sentryVersion, + templatePath: df.templatePath, + category: df.category, + dependencies: df.dependencies, + executionMode: df.executionMode, + streamingMode: df.streamingMode, + modelOverrides: df.modelOverrides, + skip: df.skip, + }; + }); + + if (options.verbose) { + console.log( + `Setting up ${frameworks.length} framework(s) with ${testDefinitions.length} test(s)\n`, + ); + } + + // Setup tests (environment + template rendering, no execution) + await orchestrator.setupTests(frameworks, testDefinitions); + process.exit(0); +} + +main().catch((error) => { + console.error("Fatal error:", error); + process.exit(1); +}); diff --git a/src/span-collector/server.ts b/src/span-collector/server.ts new file mode 100644 index 0000000..d200e77 --- /dev/null +++ b/src/span-collector/server.ts @@ -0,0 +1,213 @@ +/** + * HTTP server that collects Sentry spans from test runs + */ + +import { Hono } from 'hono'; +import { serve } from '@hono/node-server'; +import { SpanStore } from './store.js'; +import { CapturedSpan } from '../types.js'; +import * as zlib from 'zlib'; +import { promisify } from 'util'; + +const gunzip = promisify(zlib.gunzip); + +export class SpanCollector { + private app: Hono; + private server: ReturnType | null = null; + private store: SpanStore; + private port: number = 0; + private host: string = 'localhost'; + private projectIdToRunId: Map = new Map(); + + constructor(port: number = 0) { + this.port = port; // 0 = random available port + this.store = new SpanStore(); + this.app = this.createApp(); + } + + /** + * Create Hono app with routes + */ + private createApp(): Hono { + const app = new Hono(); + + // Health check endpoint + app.get('/health', (c) => { + return c.json({ status: 'ok' }); + }); + + // Sentry envelope endpoint - matches standard Sentry DSN format + // Both /api/{projectId}/envelope/ and /{projectId}/envelope are supported + app.post('/api/:projectId/envelope/', async (c) => { + return this.handleEnvelope(c); + }); + + app.post('/:projectId/envelope/', async (c) => { + return this.handleEnvelope(c); + }); + + return app; + } + + /** + * Handle Sentry envelope submission + */ + private async handleEnvelope(c: any): Promise { + try { + const projectIdStr = c.req.param('projectId'); + const projectId = parseInt(projectIdStr, 10); + + // Look up runId from projectId + const runId = this.projectIdToRunId.get(projectId); + if (!runId) { + console.warn(`Received envelope for unknown projectId: ${projectId}`); + // Accept it anyway to avoid breaking the SDK + return c.json({ status: 'ok' }); + } + + const contentEncoding = c.req.header('content-encoding'); + + // Get raw body as buffer + const arrayBuffer = await c.req.arrayBuffer(); + let body: string; + + // Handle gzip compression + if (contentEncoding === 'gzip') { + try { + const buffer = Buffer.from(arrayBuffer); + const decompressed = await gunzip(buffer); + body = decompressed.toString('utf-8'); + } catch (gzipError) { + // If decompression fails, try as plain text + console.warn('Failed to decompress gzip, trying as plain text'); + body = Buffer.from(arrayBuffer).toString('utf-8'); + } + } else { + body = Buffer.from(arrayBuffer).toString('utf-8'); + } + + // Parse envelope and extract spans + const spans = this.parseEnvelope(body); + + // Store spans + if (spans.length > 0) { + this.store.addSpans(runId, spans); + } + + return c.json({ status: 'ok' }); + } catch (error) { + console.error('Error handling envelope:', error); + return c.json({ error: 'Failed to process envelope' }, 500); + } + } + + /** + * Parse Sentry envelope format + */ + private parseEnvelope(body: string): CapturedSpan[] { + const lines = body.trim().split('\n'); + const spans: CapturedSpan[] = []; + + // Envelope format: header line, then item header + item body pairs + for (let i = 1; i < lines.length; i += 2) { + try { + const itemHeader = JSON.parse(lines[i]); + const itemBody = JSON.parse(lines[i + 1]); + + if (itemHeader.type === 'transaction' || itemHeader.type === 'span') { + // Transaction contains spans + if (itemBody.spans) { + spans.push(...itemBody.spans); + } + // The transaction itself is also a span + if (itemBody.span_id) { + spans.push(itemBody); + } + } + } catch (error) { + console.warn('Failed to parse envelope item:', error); + } + } + + return spans; + } + + /** + * Start the HTTP server + */ + async start(): Promise { + return new Promise((resolve) => { + this.server = serve( + { + fetch: this.app.fetch, + port: this.port, + hostname: this.host, + }, + (info) => { + this.port = info.port; + resolve(); + } + ); + }); + } + + /** + * Stop the HTTP server + */ + async stop(): Promise { + if (this.server) { + this.server.close(); + this.server = null; + } + } + + /** + * Register a test run + */ + registerRun(runId: string): void { + this.store.registerRun(runId); + } + + /** + * Get DSN for a test run + * + * Returns a Sentry DSN in the format: http://public@host:port/projectId + * The runId is encoded in the project ID field (using a hash to make it numeric) + */ + getDsn(runId: string): string { + // Generate a numeric project ID from runId + // Use a simple hash to convert runId to a number + let hash = 0; + for (let i = 0; i < runId.length; i++) { + hash = ((hash << 5) - hash) + runId.charCodeAt(i); + hash = hash & hash; // Convert to 32-bit integer + } + const projectId = Math.abs(hash); + + // Store mapping of projectId -> runId for later lookup + this.projectIdToRunId.set(projectId, runId); + + return `http://public@${this.host}:${this.port}/${projectId}`; + } + + /** + * Get captured spans for a test run + */ + getSpans(runId: string): CapturedSpan[] { + return this.store.getSpans(runId); + } + + /** + * Clear spans for a test run + */ + clearRun(runId: string): void { + this.store.clearRun(runId); + } + + /** + * Get server port + */ + getPort(): number { + return this.port; + } +} diff --git a/src/span-collector/store.ts b/src/span-collector/store.ts new file mode 100644 index 0000000..aaefee8 --- /dev/null +++ b/src/span-collector/store.ts @@ -0,0 +1,45 @@ +/** + * In-memory storage for captured spans + */ + +import { CapturedSpan } from '../types.js'; + +export class SpanStore { + private spans: Map = new Map(); + + /** + * Register a new test run + */ + registerRun(runId: string): void { + this.spans.set(runId, []); + } + + /** + * Add spans for a test run + */ + addSpans(runId: string, spans: CapturedSpan[]): void { + const existing = this.spans.get(runId) || []; + this.spans.set(runId, [...existing, ...spans]); + } + + /** + * Get spans for a test run + */ + getSpans(runId: string): CapturedSpan[] { + return this.spans.get(runId) || []; + } + + /** + * Clear spans for a test run + */ + clearRun(runId: string): void { + this.spans.delete(runId); + } + + /** + * Get all run IDs + */ + getRunIds(): string[] { + return Array.from(this.spans.keys()); + } +} diff --git a/src/test-cases/README.md b/src/test-cases/README.md new file mode 100644 index 0000000..cc4db99 --- /dev/null +++ b/src/test-cases/README.md @@ -0,0 +1,200 @@ +# Test Cases + +Abstract test definitions that describe expected behavior across all frameworks. + +## Structure + +``` +test-cases/ +├── llm/ # Tests for LLM-only frameworks (OpenAI, Anthropic, etc.) +│ └── basic.ts # Basic completion test +├── agents/ # Tests for agentic frameworks (Vercel AI, LangChain, etc.) +│ └── basic.ts # Basic agent with tool test +└── index.ts # Exports all test cases +``` + +## Test Definition Format + +Each test case exports a `TestDefinition` object: + +```typescript +export const basicLLMTest: TestDefinition = { + name: 'Basic LLM Test', + description: 'Single completion call with system message', + + // For LLM tests: system + input + system: 'You are a helpful assistant.', + input: { + model: 'gpt-4o', + prompt: 'What is the capital of France?', + }, + + // For agent tests: agent + input + agent: { + name: 'math_assistant', + tools: [/* ... */], + }, + + // Validation function (runs in orchestrator with Chai) + checks(spans) { + expect(spans.length).to.be.greaterThan(0); + // ... more assertions + } +}; +``` + +## LLM Tests + +Tests for frameworks that support direct LLM calls (all frameworks): + +- **`basic.ts`** - Single completion with system message + +**Compatible frameworks:** +- OpenAI SDK +- Anthropic SDK +- Vercel AI SDK +- LangChain +- Google GenAI + +## Agent Tests + +Tests for frameworks that support agentic workflows (subset of frameworks): + +- **`basic.ts`** - Agent with simple tool call + +**Compatible frameworks:** +- Vercel AI SDK (agentic mode) +- LangChain (agents) +- OpenAI Agents SDK +- LangGraph + +## Adding a New Test Case + +1. **Create test file:** + ```bash + # For LLM test + touch src/test-cases/llm/my-test.ts + + # For agent test + touch src/test-cases/agents/my-test.ts + ``` + +2. **Define test:** + ```typescript + import { TestDefinition } from '../../types.js'; + import { expect } from 'chai'; + + export const myTest: TestDefinition = { + name: 'My Test', + description: 'What this test validates', + + // System or agent config + system: '...', // OR agent: { ... } + + input: { + model: 'gpt-4o', + prompt: '...', + }, + + checks(spans) { + // Chai assertions + expect(spans.length).to.be.greaterThan(0); + } + }; + + export default myTest; + ``` + +3. **Export in index:** + ```typescript + // src/test-cases/index.ts + import { myTest } from './llm/my-test.js'; + + export const testCases = { + llm: { + basic: basicLLMTest, + myTest: myTest, // Add here + }, + // ... + }; + ``` + +## Validation Guidelines + +The `checks(spans)` function receives captured Sentry spans and should validate: + +### Basic Checks (all tests) + +```typescript +checks(spans) { + const genAISpans = spans.filter(s => s.op?.startsWith('gen_ai')); + + // Should capture spans + expect(genAISpans.length).to.be.at.least(1); + + // Basic span structure + expect(genAISpans[0].op).to.exist; + expect(genAISpans[0].start_timestamp).to.exist; + expect(genAISpans[0].timestamp).to.exist; +} +``` + +### AI Monitoring Attributes + +```typescript +// Model information +expect(span.data['gen_ai.request.model']).to.exist; + +// Token usage (if available) +if (span.data['gen_ai.usage.input_tokens']) { + expect(span.data['gen_ai.usage.input_tokens']).to.be.a('number'); +} + +// Prompt data (if captured) +if (span.data['gen_ai.prompt']) { + expect(span.data['gen_ai.prompt']).to.be.a('string'); +} +``` + +### Agent-Specific Checks + +```typescript +// Agent span exists +const agentSpan = spans.find(s => s.op === 'gen_ai.invoke_agent'); +expect(agentSpan).to.exist; + +// Tool calls captured +if (span.data['gen_ai.tool_calls']) { + expect(span.data['gen_ai.tool_calls']).to.be.an('array'); +} +``` + +## Framework Compatibility + +Test cases are automatically filtered based on framework type: + +| Framework Type | Compatible Tests | +|----------------|------------------| +| `llm-only` | Only tests with `system` property | +| `agentic` | Tests with `system` OR `agent` property | + +The orchestrator handles this filtering automatically. + +## Test Execution Flow + +1. Orchestrator loads test definitions +2. Generates test matrix (framework × compatible tests) +3. For each test: + - Renders framework template with test inputs + - Executes test in isolated environment + - Collects Sentry spans + - Runs `checks(spans)` function + - Reports pass/fail + +## Tips + +- **Keep tests focused** - Test one behavior per test case +- **Use descriptive names** - Make it clear what's being tested +- **Flexible assertions** - Account for framework differences +- **Log useful info** - Use `console.log` to show what was captured +- **Handle missing data** - Not all SDKs capture everything diff --git a/src/test-cases/agents/basic.ts b/src/test-cases/agents/basic.ts new file mode 100644 index 0000000..b039ab5 --- /dev/null +++ b/src/test-cases/agents/basic.ts @@ -0,0 +1,52 @@ +/** + * Basic Agent Test Case + * + * Tests an agentic workflow WITHOUT tools - similar to the basic LLM test. + * Validates that Sentry captures agent spans correctly for simple completions. + */ + +import { TestDefinition } from "../../types.js"; +import { + checkChatSpanAttributes, + checkAgentSpanAttributes, + checkValidTokenUsage, + checkAgentHierarchy, + checkInputTokensCached, + checkOutputTokensReasoning, + checkInputMessagesSchema, +} from "../checks.js"; + +export const basicAgentTest: TestDefinition = { + name: "Basic Agent Test", + description: "Agent without tools - simple completion", + type: "agent", + + // Agent without tools - just a simple assistant + agent: { + name: "helpful_assistant", + description: "A helpful assistant that answers questions", + tools: [], + }, + + inputs: [ + { + model: "gpt-4o-mini", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "What is the capital of France?" }, + ], + }, + ], + + checks: [ + checkAgentSpanAttributes, + checkChatSpanAttributes, + checkValidTokenUsage, + checkAgentHierarchy, + checkInputMessagesSchema, + checkInputTokensCached, + checkOutputTokensReasoning, + ], +}; + +export default basicAgentTest; diff --git a/src/test-cases/agents/long-input.ts b/src/test-cases/agents/long-input.ts new file mode 100644 index 0000000..2e69148 --- /dev/null +++ b/src/test-cases/agents/long-input.ts @@ -0,0 +1,82 @@ +/** + * Long Input Agent Test Case + * + * Tests that very long user messages (>20KB) trigger proper trimming + * of gen_ai.request.messages in the Sentry span data for agentic frameworks. + * + * Sentry SDKs trim long messages to prevent excessive span sizes. + * This test validates: + * 1. The message content is trimmed (less than original size) + * 2. Metadata about trimming is present (original_length) + * 3. Basic attributes are still captured correctly + * 4. Token counts reflect the actual (untrimmed) input + */ + +import { TestDefinition } from "../../types.js"; +import { + checkChatSpanAttributes, + checkAgentSpanAttributes, + checkMessageTrimming, + checkTrimmingMetadata, + checkAgentHierarchy, + checkInputMessagesSchema, +} from "../checks.js"; + +// Generate a long message that exceeds 20KB +// We'll repeat a pattern to create predictable content +const LONG_MESSAGE_PATTERN = + "This is a test message that will be repeated many times to create a very long input. "; +const REPETITIONS = 300; // ~25KB of text (85 chars * 300 = 25,500 bytes) +const LONG_MESSAGE = LONG_MESSAGE_PATTERN.repeat(REPETITIONS); + +export const longInputAgentTest: TestDefinition = { + name: "Long Input Agent Test", + description: "Tests message trimming for agent inputs > 20KB", + type: "agent", + + // Simple agent with a tool - the focus is on the long input, not tool usage + agent: { + name: "summarizer_assistant", + description: "An assistant that can summarize text", + tools: [ + { + name: "get_word_count", + description: "Count the number of words in a text", + parameters: { + type: "object", + properties: { + text: { + type: "string", + description: "The text to count words in", + }, + }, + required: ["text"], + }, + result: 2400, // Approximate word count for our long message + }, + ], + }, + + inputs: [ + { + model: "gpt-4o-mini", + messages: [ + { + role: "user", + content: `Please summarize the following text in one sentence. You may use the get_word_count tool first if needed: ${LONG_MESSAGE}`, + }, + ], + }, + ], + + checks: [ + checkAgentSpanAttributes, + checkChatSpanAttributes, + checkMessageTrimming, + checkTrimmingMetadata, + checkAgentHierarchy, + checkInputMessagesSchema, + ], +}; + +export default longInputAgentTest; diff --git a/src/test-cases/agents/tool-call.ts b/src/test-cases/agents/tool-call.ts new file mode 100644 index 0000000..987c914 --- /dev/null +++ b/src/test-cases/agents/tool-call.ts @@ -0,0 +1,123 @@ +/** + * Tool Call Agent Test Case + * + * Tests an agentic workflow with multiple tool calls. + * Validates that Sentry captures both agent and tool call spans correctly. + * + * Expression: (3 + 5) * 4 = 32 + * - First call: add(3, 5) = 8 + * - Second call: multiply(8, 4) = 32 + */ + +import { TestDefinition } from "../../types.js"; +import { + checkChatSpanAttributes, + checkAgentSpanAttributes, + checkToolSpanAttributes, + checkValidTokenUsage, + checkAgentHierarchy, + checkInputTokensCached, + checkOutputTokensReasoning, + checkToolCalls, + checkAvailableTools, + checkResponseToolCalls, + checkInputMessagesSchema, +} from "../checks.js"; + +export const toolCallAgentTest: TestDefinition = { + name: "Tool Call Agent Test", + description: "Agent with multiple tool calls", + type: "agent", + + agent: { + name: "math_assistant", + description: "A math assistant that can perform basic arithmetic", + tools: [ + { + name: "add", + description: "Add two numbers together", + parameters: { + type: "object", + properties: { + a: { + type: "number", + description: "First number", + }, + b: { + type: "number", + description: "Second number", + }, + }, + required: ["a", "b"], + }, + result: 8, // Static result: 3 + 5 = 8 + }, + { + name: "multiply", + description: "Multiply two numbers together", + parameters: { + type: "object", + properties: { + a: { + type: "number", + description: "First number", + }, + b: { + type: "number", + description: "Second number", + }, + }, + required: ["a", "b"], + }, + result: 32, // Static result: 8 * 4 = 32 + }, + ], + }, + + inputs: [ + { + model: "gpt-4o-mini", + messages: [ + { + role: "user", + content: + "Calculate (3 + 5) * 4. First use the add tool to add 3 and 5, then use the multiply tool to multiply the result by 4.", + }, + ], + }, + ], + + checks: [ + checkAgentSpanAttributes, + checkChatSpanAttributes, + checkToolSpanAttributes, + checkValidTokenUsage, + checkAgentHierarchy, + checkAvailableTools, + checkResponseToolCalls([ + { name: "add", arguments: { a: 3, b: 5 } }, + { name: "multiply", arguments: { a: 8, b: 4 } }, + ]), + checkToolCalls([ + { + name: "add", + type: "function", + description: "Add two numbers together", + input: { a: 3, b: 5 }, + output: 8, + }, + { + name: "multiply", + type: "function", + description: "Multiply two numbers together", + input: { a: 8, b: 4 }, + output: 32, + }, + ]), + checkInputMessagesSchema, + checkInputTokensCached, + checkOutputTokensReasoning, + ], +}; + +export default toolCallAgentTest; diff --git a/src/test-cases/agents/tool-error.ts b/src/test-cases/agents/tool-error.ts new file mode 100644 index 0000000..e260f53 --- /dev/null +++ b/src/test-cases/agents/tool-error.ts @@ -0,0 +1,116 @@ +/** + * Tool Error Agent Test Case + * + * Tests an agentic workflow where a tool raises an exception. + * Validates that Sentry captures the error correctly in spans. + */ + +import { expect } from "chai"; +import { TestDefinition, CapturedSpan, Check } from "../../types.js"; +import { + checkChatSpanAttributes, + checkAgentSpanAttributes, + checkToolSpanAttributes, + checkAgentHierarchy, + checkInputMessagesSchema, + checkAvailableTools, + checkResponseToolCalls, +} from "../checks.js"; +import { extractGenAISpans, findToolSpans } from "../utils.js"; + +/** + * Check that a tool error was captured in spans + */ +const checkToolErrorSpan: Check = { + name: "checkToolErrorSpan", + fn: (spans: CapturedSpan[], config, testDef) => { + const toolSpans = findToolSpans(extractGenAISpans(spans)); + expect( + toolSpans.length, + "Should have at least one tool span", + ).to.be.greaterThan(0); + + // Get expected tool name from test definition + const expectedToolName = testDef.agent?.tools?.[0]?.name; + expect(expectedToolName, "Test should define at least one tool").to.exist; + + // Find the span for the expected tool + const toolSpan = toolSpans.find( + (s) => + s.data?.["gen_ai.tool.name"] === expectedToolName || + s.description?.includes(expectedToolName!), + ); + expect(toolSpan, `Should have a span for tool "${expectedToolName}"`).to + .exist; + + // Check for error indicators + const span = toolSpan!; + const hasError = + span.status === "error" || + span.status === "internal_error" || + span.data?.["error"] !== undefined || + span.data?.["exception"] !== undefined || + span.data?.["gen_ai.tool.error"] !== undefined || + (span.tags && span.tags["error"] === true); + + expect(hasError, "Tool span should have an error indicator").to.be.true; + }, +}; + +export const toolErrorAgentTest: TestDefinition = { + name: "Tool Error Agent Test", + description: "Agent with tool that raises an exception", + type: "agent", + + agent: { + name: "file_assistant", + description: "An assistant that can read files", + tools: [ + { + name: "read_file", + description: "Read the contents of a file", + parameters: { + type: "object", + properties: { + path: { + type: "string", + description: "The path to the file to read", + }, + }, + required: ["path"], + }, + // This tool will raise an error instead of returning a result + error: + "FileNotFoundError: The file '/nonexistent/file.txt' does not exist", + }, + ], + }, + + inputs: [ + { + model: "gpt-4o-mini", + messages: [ + { + role: "user", + content: + "Please read the file at /nonexistent/file.txt and tell me what it contains. Use the read_file tool.", + }, + ], + }, + ], + + checks: [ + checkAgentSpanAttributes, + checkChatSpanAttributes, + checkToolSpanAttributes, + checkAgentHierarchy, + checkAvailableTools, + checkResponseToolCalls([ + { name: "read_file", arguments: { path: "/nonexistent/file.txt" } }, + ]), + checkInputMessagesSchema, + checkToolErrorSpan, + ], +}; + +export default toolErrorAgentTest; diff --git a/src/test-cases/agents/vision.ts b/src/test-cases/agents/vision.ts new file mode 100644 index 0000000..0347ead --- /dev/null +++ b/src/test-cases/agents/vision.ts @@ -0,0 +1,72 @@ +/** + * Vision Agent Test Case + * + * Tests an agentic workflow with image input. + * Validates that Sentry captures agent spans when processing images. + */ + +import { TestDefinition } from "../../types.js"; +import { + checkChatSpanAttributes, + checkAgentSpanAttributes, + checkValidTokenUsage, + checkAgentHierarchy, + checkInputMessagesSchema, + checkBinaryRedaction, +} from "../checks.js"; + +// Small 10x10 red PNG image encoded as base64 +const TEST_IMAGE_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8z8BQz0AEYBxVSF+FABJADveWkH6oAAAAAElFTkSuQmCC"; + +export const visionAgentTest: TestDefinition = { + name: "Vision Agent Test", + description: "Agent that analyzes an image", + type: "agent", + + // No tools needed for this test - just image analysis + agent: { + name: "vision_assistant", + description: + "An assistant that can analyze images and describe what it sees", + tools: [], + }, + + inputs: [ + { + model: "gpt-4o-mini", + messages: [ + { + role: "system", + content: + "You are a helpful assistant that can analyze images. Be concise.", + }, + { + role: "user", + content: [ + { + type: "text", + text: "What color is this image? Reply with just the color name.", + }, + { + type: "image", + base64: TEST_IMAGE_BASE64, + mediaType: "image/png", + }, + ], + }, + ], + }, + ], + + checks: [ + checkAgentSpanAttributes, + checkChatSpanAttributes, + checkValidTokenUsage, + checkAgentHierarchy, + checkInputMessagesSchema, + checkBinaryRedaction, + ], +}; + +export default visionAgentTest; diff --git a/src/test-cases/checks.ts b/src/test-cases/checks.ts new file mode 100644 index 0000000..08d487d --- /dev/null +++ b/src/test-cases/checks.ts @@ -0,0 +1,1148 @@ +/** + * Reusable check functions for test cases + * + * Each check function follows the signature: + * (spans: CapturedSpan[], config: FrameworkConfig, testDef: TestDefinition) => void + * + * Check functions can: + * - Throw an error to fail the check + * - Call skip() or skipIf() to skip the check + * - Use expect() from chai for assertions + */ + +import { expect } from "chai"; +import { CapturedSpan, FrameworkConfig, TestDefinition } from "../types.js"; +import { + extractGenAISpans, + findAgentSpans, + findChatSpans, + findToolSpans, + findHandoffSpans, + assertAttributes, + checkTokenUsage, + skip, + skipIf, +} from "./utils.js"; + +/** + * Check function signature + */ +export type CheckFunction = ( + spans: CapturedSpan[], + config: FrameworkConfig, + testDef: TestDefinition, +) => void | Promise; + +/** + * Check definition with name and function + */ +export interface Check { + name: string; + fn: CheckFunction; +} + +// ============================================================================= +// Structure Checks +// ============================================================================= + +/** + * Factory function to create a check that validates the number of AI spans + * + * @param expected - The expected number of AI spans, or an object with min/max bounds + * @returns A Check object that validates the span count + * + * @example + * // Exactly 1 span + * checkAISpanCount(1) + * + * @example + * // At least 1 span + * checkAISpanCount({ min: 1 }) + * + * @example + * // Between 2 and 5 spans + * checkAISpanCount({ min: 2, max: 5 }) + * + * @example + * // At most 3 spans + * checkAISpanCount({ max: 3 }) + */ +export function checkAISpanCount( + expected: number | { min?: number; max?: number }, +): Check { + // Determine name based on expected value + let name: string; + if (typeof expected === "number") { + name = `checkAISpanCount(${expected})`; + } else if (expected.min !== undefined && expected.max !== undefined) { + name = `checkAISpanCount(${expected.min}-${expected.max})`; + } else if (expected.min !== undefined) { + name = `checkAISpanCount(>=${expected.min})`; + } else if (expected.max !== undefined) { + name = `checkAISpanCount(<=${expected.max})`; + } else { + name = "checkAISpanCount"; + } + + return { + name, + fn: (spans) => { + const aiSpans = extractGenAISpans(spans); + + if (typeof expected === "number") { + // Exact count + expect( + aiSpans.length, + `Should have exactly ${expected} AI span(s)`, + ).to.equal(expected); + } else { + // Range check + if (expected.min !== undefined) { + expect( + aiSpans.length, + `Should have at least ${expected.min} AI span(s)`, + ).to.be.at.least(expected.min); + } + if (expected.max !== undefined) { + expect( + aiSpans.length, + `Should have at most ${expected.max} AI span(s)`, + ).to.be.at.most(expected.max); + } + } + }, + }; +} + +// ============================================================================= +// Span Type Attribute Checks +// ============================================================================= + +/** + * Check attributes on chat/completion spans (LLM API calls) + * + * Validates: + * - gen_ai.operation.name exists + * - gen_ai.request.model matches expected model + * - gen_ai.request.messages exists + * - gen_ai.response.model matches expected pattern + * - gen_ai.response.text exists + * - gen_ai.usage.input_tokens exists + * - gen_ai.usage.output_tokens exists + * + * Fails if no chat spans are found. + */ +export const checkChatSpanAttributes: Check = { + name: "checkChatSpanAttributes", + fn: (spans, config, testDef) => { + const chatSpans = findChatSpans(extractGenAISpans(spans)); + expect( + chatSpans.length, + "Should have at least one chat/completion span", + ).to.be.greaterThan(0); + + const requestModel = + config.modelOverrides?.request || testDef.inputs[0]?.model || "gpt-*"; + const responseModel = + config.modelOverrides?.response || `${requestModel.replace("*", "")}*`; + + assertAttributes(chatSpans, { + "gen_ai.operation.name": true, + "gen_ai.request.model": requestModel, + "gen_ai.request.messages": true, + "gen_ai.response.model": responseModel, + "gen_ai.response.text": true, + "gen_ai.usage.input_tokens": true, + "gen_ai.usage.output_tokens": true, + }); + }, +}; + +/** + * Check attributes on invoke_agent spans (agent invocations) + * + * Validates: + * - gen_ai.agent.name exists + * + * Fails if no agent spans are found. + */ +export const checkAgentSpanAttributes: Check = { + name: "checkAgentSpanAttributes", + fn: (spans) => { + const agentSpans = findAgentSpans(extractGenAISpans(spans)); + expect( + agentSpans.length, + "Should have at least one agent span", + ).to.be.greaterThan(0); + + // TODO: Add attribute validation once we know what attributes agent spans should have + for (const span of agentSpans) { + expect( + span.data?.["gen_ai.agent.name"], + `Agent span should have gen_ai.agent.name attribute`, + ).to.exist; + } + }, +}; + +/** + * Check attributes on tool execution spans + * + * Validates: + * - gen_ai.tool.type exists + * - gen_ai.tool.name exists + * - gen_ai.tool.description exists + * + * Fails if no tool spans are found. + */ +export const checkToolSpanAttributes: Check = { + name: "checkToolSpanAttributes", + fn: (spans) => { + const toolSpans = findToolSpans(extractGenAISpans(spans)); + expect( + toolSpans.length, + "Should have at least one tool span", + ).to.be.greaterThan(0); + + for (const span of toolSpans) { + expect( + span.data?.["gen_ai.tool.type"], + `Tool span should have gen_ai.tool.type attribute`, + ).to.exist; + expect( + span.data?.["gen_ai.tool.name"], + `Tool span should have gen_ai.tool.name attribute`, + ).to.exist; + expect( + span.data?.["gen_ai.tool.description"], + `Tool span should have gen_ai.tool.description attribute`, + ).to.exist; + } + }, +}; + +/** + * Expected tool call definition for validation + */ +export interface ExpectedToolCall { + /** Tool name to match */ + name: string; + /** Expected tool type (e.g., "function") */ + type?: string; + /** Expected tool description */ + description?: string; + /** Expected input arguments (parsed from gen_ai.tool.input JSON) */ + input?: Record; + /** Expected output value */ + output?: unknown; +} + +/** + * Factory function to create a check that validates specific tool calls + * + * @param expectedTools - Array of expected tool calls to validate + * @returns A Check object that validates the tool calls + * + * @example + * // Check a single tool call + * checkToolCalls([{ + * name: "add", + * type: "function", + * description: "Add two numbers together", + * input: { a: 4, b: 7 }, + * output: 11, + * }]) + * + * @example + * // Check multiple tool calls + * checkToolCalls([ + * { name: "search", input: { query: "weather" } }, + * { name: "format", input: { data: "..." } }, + * ]) + */ +export function checkToolCalls(expectedTools: ExpectedToolCall[]): Check { + const toolNames = expectedTools.map((t) => t.name).join(", "); + return { + name: `checkToolCalls(${toolNames})`, + fn: (spans) => { + const toolSpans = findToolSpans(extractGenAISpans(spans)); + expect( + toolSpans.length, + `Should have at least ${expectedTools.length} tool span(s)`, + ).to.be.at.least(expectedTools.length); + + for (const expected of expectedTools) { + // Find the tool span matching this expected tool + const toolSpan = toolSpans.find( + (s) => s.data?.["gen_ai.tool.name"] === expected.name, + ); + expect(toolSpan, `Should have a tool span for "${expected.name}"`).to + .exist; + + const span = toolSpan!; + + // Validate type if specified + if (expected.type !== undefined) { + expect( + span.data?.["gen_ai.tool.type"], + `Tool "${expected.name}" should have type "${expected.type}"`, + ).to.equal(expected.type); + } + + // Validate description if specified + if (expected.description !== undefined) { + expect( + span.data?.["gen_ai.tool.description"], + `Tool "${expected.name}" should have description`, + ).to.equal(expected.description); + } + + // Validate input if specified + if (expected.input !== undefined) { + const inputRaw = span.data?.["gen_ai.tool.input"]; + expect( + inputRaw, + `Tool "${expected.name}" should have gen_ai.tool.input`, + ).to.exist; + + // Parse input if it's a JSON string + let input: Record; + if (typeof inputRaw === "string") { + try { + input = JSON.parse(inputRaw); + } catch { + throw new Error( + `Tool "${expected.name}" has invalid JSON in gen_ai.tool.input: ${inputRaw}`, + ); + } + } else { + input = inputRaw as Record; + } + + // Check each expected input field + for (const [key, value] of Object.entries(expected.input)) { + expect( + input[key], + `Tool "${expected.name}" input should have "${key}"`, + ).to.exist; + // If a specific value is expected, check it (convert to same type for comparison) + if (value !== undefined) { + const actualValue = input[key]; + // Handle numeric string comparison (some frameworks pass numbers as strings) + if ( + typeof value === "number" && + typeof actualValue === "string" + ) { + expect( + Number(actualValue), + `Tool "${expected.name}" input.${key} should equal ${value}`, + ).to.equal(value); + } else { + expect( + actualValue, + `Tool "${expected.name}" input.${key} should equal ${JSON.stringify(value)}`, + ).to.deep.equal(value); + } + } + } + } + + // Validate output if specified + if (expected.output !== undefined) { + const outputRaw = span.data?.["gen_ai.tool.output"]; + expect( + outputRaw, + `Tool "${expected.name}" should have gen_ai.tool.output`, + ).to.exist; + + // Parse output if it's a JSON string + let output: unknown; + if (typeof outputRaw === "string") { + try { + output = JSON.parse(outputRaw); + } catch { + // Not JSON, use raw value + output = outputRaw; + } + } else { + output = outputRaw; + } + + expect( + output, + `Tool "${expected.name}" output should equal ${JSON.stringify(expected.output)}`, + ).to.deep.equal(expected.output); + } + } + }, + }; +} + +/** + * Check that gen_ai.request.available_tools matches the tools defined in the test + * + * Validates that chat spans contain available_tools that match the agent's tool definitions. + * Checks tool name, description, and parameter schema. + */ +export const checkAvailableTools: Check = { + name: "checkAvailableTools", + fn: (spans, config, testDef) => { + const chatSpans = findChatSpans(extractGenAISpans(spans)); + expect( + chatSpans.length, + "Should have at least one chat span", + ).to.be.greaterThan(0); + + const definedTools = testDef.agent?.tools || []; + expect( + definedTools.length, + "Test should define at least one tool", + ).to.be.greaterThan(0); + + // Find a chat span with available_tools + const spanWithTools = chatSpans.find( + (s) => s.data?.["gen_ai.request.available_tools"] !== undefined, + ); + expect( + spanWithTools, + "Should have a chat span with gen_ai.request.available_tools", + ).to.exist; + + const availableToolsRaw = + spanWithTools!.data?.["gen_ai.request.available_tools"]; + + // Parse if JSON string + let availableTools: Array>; + if (typeof availableToolsRaw === "string") { + try { + availableTools = JSON.parse(availableToolsRaw); + } catch { + throw new Error( + `Invalid JSON in gen_ai.request.available_tools: ${availableToolsRaw}`, + ); + } + } else { + availableTools = availableToolsRaw as Array>; + } + + expect( + Array.isArray(availableTools), + "gen_ai.request.available_tools should be an array", + ).to.be.true; + + // Check each defined tool exists in available_tools + for (const definedTool of definedTools) { + const foundTool = availableTools.find((t) => { + // Tools can be nested under "function" key or at top level + const toolName = + t.name || (t.function as Record)?.name; + return toolName === definedTool.name; + }); + + expect(foundTool, `Available tools should include "${definedTool.name}"`) + .to.exist; + + // Check description if present + const toolDesc = + foundTool!.description || + (foundTool!.function as Record)?.description; + if (definedTool.description) { + expect( + toolDesc, + `Tool "${definedTool.name}" should have description`, + ).to.equal(definedTool.description); + } + } + + // Check count matches + expect( + availableTools.length, + `Should have ${definedTools.length} available tool(s)`, + ).to.equal(definedTools.length); + }, +}; + +/** + * Expected tool call in gen_ai.response.tool_calls + */ +export interface ExpectedResponseToolCall { + /** Tool name to match */ + name: string; + /** Expected arguments (id fields are ignored) */ + arguments: Record; +} + +/** + * Factory function to check gen_ai.response.tool_calls on chat spans + * + * Validates that a chat span contains tool_calls with the expected tool names + * and arguments. Tool call IDs are ignored since they're generated dynamically. + * + * @param expectedToolCalls - Array of expected tool calls + * @returns A Check object that validates the response tool calls + * + * @example + * checkResponseToolCalls([ + * { name: "add", arguments: { a: 3, b: 5 } }, + * { name: "multiply", arguments: { a: 8, b: 4 } }, + * ]) + */ +export function checkResponseToolCalls( + expectedToolCalls: ExpectedResponseToolCall[], +): Check { + const toolNames = expectedToolCalls.map((t) => t.name).join(", "); + return { + name: `checkResponseToolCalls(${toolNames})`, + fn: (spans) => { + const chatSpans = findChatSpans(extractGenAISpans(spans)); + expect( + chatSpans.length, + "Should have at least one chat span", + ).to.be.greaterThan(0); + + // Collect all tool_calls from all chat spans + const allToolCalls: Array> = []; + + for (const span of chatSpans) { + const toolCallsRaw = span.data?.["gen_ai.response.tool_calls"]; + if (toolCallsRaw === undefined) continue; + + // Parse if JSON string + let toolCalls: Array>; + if (typeof toolCallsRaw === "string") { + try { + toolCalls = JSON.parse(toolCallsRaw); + } catch { + throw new Error( + `Invalid JSON in gen_ai.response.tool_calls: ${toolCallsRaw}`, + ); + } + } else { + toolCalls = toolCallsRaw as Array>; + } + + if (Array.isArray(toolCalls)) { + allToolCalls.push(...toolCalls); + } + } + + expect( + allToolCalls.length, + `Should have at least ${expectedToolCalls.length} tool call(s) in response`, + ).to.be.at.least(expectedToolCalls.length); + + // Check each expected tool call + for (const expected of expectedToolCalls) { + // Find matching tool call by name + const foundCall = allToolCalls.find((tc) => { + // Handle different formats: { name, arguments } or { function: { name, arguments } } + const tcName = + tc.name || (tc.function as Record)?.name; + return tcName === expected.name; + }); + + expect( + foundCall, + `Response should include tool call for "${expected.name}"`, + ).to.exist; + + // Get arguments + let actualArgs: Record; + const argsRaw = + foundCall!.arguments || + (foundCall!.function as Record)?.arguments; + + if (typeof argsRaw === "string") { + try { + actualArgs = JSON.parse(argsRaw); + } catch { + throw new Error( + `Invalid JSON in tool call arguments for "${expected.name}": ${argsRaw}`, + ); + } + } else { + actualArgs = (argsRaw as Record) || {}; + } + + // Check each expected argument + for (const [key, value] of Object.entries(expected.arguments)) { + expect( + actualArgs[key], + `Tool call "${expected.name}" should have argument "${key}"`, + ).to.exist; + + const actualValue = actualArgs[key]; + // Handle numeric string comparison + if (typeof value === "number" && typeof actualValue === "string") { + expect( + Number(actualValue), + `Tool call "${expected.name}" argument "${key}" should equal ${value}`, + ).to.equal(value); + } else { + expect( + actualValue, + `Tool call "${expected.name}" argument "${key}" should equal ${JSON.stringify(value)}`, + ).to.deep.equal(value); + } + } + } + }, + }; +} + +// ============================================================================= +// Message Schema Checks +// ============================================================================= + +/** + * Valid roles for messages in gen_ai.input.messages + */ +const VALID_MESSAGE_ROLES = ["user", "assistant", "tool", "system"] as const; + +/** + * Valid part types for message parts + */ +const VALID_PART_TYPES = [ + "text", + "tool_call", + "tool_call_response", + "image", +] as const; + +/** + * Check that gen_ai.input.messages on chat spans follows the expected schema + * + * Schema (from Sentry conventions): + * - Must be a stringified array of message objects + * - Each message must have a "role" field: "user", "assistant", "tool", or "system" + * - Each message must have a "parts" array (new format) or "content" field (legacy) + * - Parts can have types: "text", "tool_call", "tool_call_response", "image" + * + * This check validates schema structure, not actual content. + */ +export const checkInputMessagesSchema: Check = { + name: "checkInputMessagesSchema", + fn: (spans) => { + const chatSpans = findChatSpans(extractGenAISpans(spans)); + const agentSpans = findAgentSpans(extractGenAISpans(spans)); + const spansToCheck = [...chatSpans, ...agentSpans]; + + expect( + spansToCheck.length, + "Should have at least one chat or agent span", + ).to.be.greaterThan(0); + + let foundMessages = false; + + for (const span of spansToCheck) { + // Check both new format (gen_ai.input.messages) and legacy (gen_ai.request.messages) + const messagesRaw = + span.data?.["gen_ai.input.messages"] ?? + span.data?.["gen_ai.request.messages"]; + + if (messagesRaw === undefined) continue; + foundMessages = true; + + // Parse if JSON string + let messages: unknown[]; + if (typeof messagesRaw === "string") { + try { + messages = JSON.parse(messagesRaw); + } catch { + throw new Error( + `Invalid JSON in gen_ai.input.messages: ${messagesRaw.substring(0, 100)}...`, + ); + } + } else { + messages = messagesRaw as unknown[]; + } + + expect( + Array.isArray(messages), + "gen_ai.input.messages should be an array", + ).to.be.true; + + expect( + messages.length, + "gen_ai.input.messages should not be empty", + ).to.be.greaterThan(0); + + // Validate each message + for (let i = 0; i < messages.length; i++) { + const msg = messages[i] as Record; + const msgPath = `messages[${i}]`; + + expect( + typeof msg === "object" && msg !== null, + `${msgPath} should be an object`, + ).to.be.true; + + // Check role + expect(msg.role, `${msgPath} should have a role field`).to.exist; + expect( + VALID_MESSAGE_ROLES.includes( + msg.role as (typeof VALID_MESSAGE_ROLES)[number], + ), + `${msgPath}.role should be one of: ${VALID_MESSAGE_ROLES.join(", ")} (got: ${msg.role})`, + ).to.be.true; + + // Check for content (parts array or content string/array) + const hasParts = msg.parts !== undefined; + const hasContent = msg.content !== undefined; + + expect( + hasParts || hasContent, + `${msgPath} should have either "parts" or "content" field`, + ).to.be.true; + + // Validate parts array if present + if (hasParts) { + expect( + Array.isArray(msg.parts), + `${msgPath}.parts should be an array`, + ).to.be.true; + + const parts = msg.parts as Array>; + for (let j = 0; j < parts.length; j++) { + const part = parts[j]; + const partPath = `${msgPath}.parts[${j}]`; + + expect( + typeof part === "object" && part !== null, + `${partPath} should be an object`, + ).to.be.true; + + // Parts should have a type + if (part.type !== undefined) { + expect( + VALID_PART_TYPES.includes( + part.type as (typeof VALID_PART_TYPES)[number], + ), + `${partPath}.type should be one of: ${VALID_PART_TYPES.join(", ")} (got: ${part.type})`, + ).to.be.true; + + // Validate type-specific fields + if (part.type === "text") { + expect( + part.text !== undefined || part.content !== undefined, + `${partPath} with type "text" should have "text" or "content" field`, + ).to.be.true; + } else if (part.type === "tool_call") { + expect( + part.name, + `${partPath} with type "tool_call" should have "name" field`, + ).to.exist; + } else if (part.type === "tool_call_response") { + expect( + part.id !== undefined || part.tool_call_id !== undefined, + `${partPath} with type "tool_call_response" should have "id" or "tool_call_id" field`, + ).to.be.true; + } + } + } + } + + // Validate content if present (can be string or array) + if (hasContent && !hasParts) { + const content = msg.content; + const isValidContent = + typeof content === "string" || + Array.isArray(content) || + (typeof content === "object" && content !== null); + + expect( + isValidContent, + `${msgPath}.content should be a string, array, or object`, + ).to.be.true; + } + } + } + + expect( + foundMessages, + "Should have at least one span with gen_ai.input.messages or gen_ai.request.messages", + ).to.be.true; + }, +}; + +/** + * Check that binary content (images) is redacted in gen_ai.request.messages + * + * Sentry SDKs should replace binary data (base64 images, etc.) with "[Blob substitute]" + * to avoid capturing large binary payloads in span data. + * + * This check validates that: + * - Messages attribute exists + * - Messages contain "[Blob substitute]" marker (indicating redaction occurred) + * - No raw base64 data is present (would indicate redaction failed) + */ +export const checkBinaryRedaction: Check = { + name: "checkBinaryRedaction", + fn: (spans) => { + const chatSpans = findChatSpans(extractGenAISpans(spans)); + const agentSpans = findAgentSpans(extractGenAISpans(spans)); + const spansToCheck = [...chatSpans, ...agentSpans]; + + expect( + spansToCheck.length, + "Should have at least one chat or agent span", + ).to.be.greaterThan(0); + + let foundMessages = false; + let foundRedaction = false; + + for (const span of spansToCheck) { + const messagesRaw = + span.data?.["gen_ai.input.messages"] ?? + span.data?.["gen_ai.request.messages"]; + + if (messagesRaw === undefined) continue; + foundMessages = true; + + // Convert to string for searching + const messagesStr = + typeof messagesRaw === "string" + ? messagesRaw + : JSON.stringify(messagesRaw); + + // Check for redaction marker + if (messagesStr.includes("[Blob substitute]")) { + foundRedaction = true; + } + + // Check that no raw base64 data is present (would be very long strings) + // Base64 image data is typically 100+ characters of alphanumeric + const base64Pattern = /[A-Za-z0-9+/]{100,}={0,2}/; + expect( + base64Pattern.test(messagesStr), + "Messages should not contain raw base64 data (should be redacted)", + ).to.be.false; + } + + expect( + foundMessages, + "Should have at least one span with messages attribute", + ).to.be.true; + + expect( + foundRedaction, + "Messages should contain '[Blob substitute]' marker indicating binary content was redacted", + ).to.be.true; + }, +}; + +/** + * Check attributes on handoff spans (agent-to-agent handoffs) + * + * Validates: + * - Handoff spans exist + * + * Fails if no handoff spans are found. + */ +export const checkHandoffSpanAttributes: Check = { + name: "checkHandoffSpanAttributes", + fn: (spans) => { + const handoffSpans = findHandoffSpans(extractGenAISpans(spans)); + expect( + handoffSpans.length, + "Should have at least one handoff span", + ).to.be.greaterThan(0); + + // TODO: Add attribute validation once we know what attributes handoff spans should have + }, +}; + +// ============================================================================= +// Token Checks +// ============================================================================= + +/** + * Check token usage on invoke_agent and ai_client spans + * Tool spans don't have token usage attributes + */ +export const checkValidTokenUsage: Check = { + name: "checkValidTokenUsage", + fn: (spans) => { + const aiSpans = extractGenAISpans(spans); + skipIf(aiSpans.length === 0, "No AI spans captured"); + + // Only check token usage on spans that should have it (not tool spans) + const tokenSpans = aiSpans.filter( + (s) => + s.op?.match(/^gen_ai\.(invoke_agent|chat|completion|generate)/) || + s.data?.["gen_ai.usage.input_tokens"] !== undefined, + ); + skipIf(tokenSpans.length === 0, "No spans with token usage"); + + for (const span of tokenSpans) { + checkTokenUsage(span, { validateSum: true }); + } + }, +}; + +/** + * Check that input tokens cached is valid when present + */ +export const checkInputTokensCached: Check = { + name: "checkInputTokensCached", + fn: (spans) => { + const aiSpans = extractGenAISpans(spans).filter( + (span) => span.data?.["gen_ai.usage.input_tokens.cached"] !== undefined, + ); + skipIf( + aiSpans.length === 0, + "No AI spans with input_tokens.cached attribute", + ); + + for (const span of aiSpans) { + expect( + span.data?.["gen_ai.usage.input_tokens.cached"], + ).to.be.lessThanOrEqual(span.data?.["gen_ai.usage.input_tokens"]); + } + }, +}; + +/** + * Check that output tokens reasoning is valid when present + */ +export const checkOutputTokensReasoning: Check = { + name: "checkOutputTokensReasoning", + fn: (spans) => { + const aiSpans = extractGenAISpans(spans).filter( + (span) => + span.data?.["gen_ai.usage.output_tokens.reasoning"] !== undefined, + ); + skipIf( + aiSpans.length === 0, + "No AI spans with output_tokens.reasoning attribute", + ); + + for (const span of aiSpans) { + expect( + span.data?.["gen_ai.usage.output_tokens.reasoning"], + ).to.be.lessThanOrEqual(span.data?.["gen_ai.usage.output_tokens"]); + } + }, +}; + +// ============================================================================= +// Message Trimming Checks +// ============================================================================= + +/** + * Check that long messages are trimmed in span data + */ +export const checkMessageTrimming: Check = { + name: "checkMessageTrimming", + fn: (spans) => { + const aiSpans = extractGenAISpans(spans); + skipIf(aiSpans.length === 0, "No AI spans captured"); + + // Find spans with message attribute + let foundTrimmedMessage = false; + const maxExpectedSize = 15000; // Sentry typically trims to ~10KB + + for (const span of aiSpans) { + const messageValue = span.data?.["gen_ai.request.messages"]; + if (messageValue !== undefined) { + const messageStr = + typeof messageValue === "string" + ? messageValue + : JSON.stringify(messageValue); + + expect(messageStr.length, "Message should be trimmed").to.be.lessThan( + maxExpectedSize, + ); + + foundTrimmedMessage = true; + } + } + + skipIf(!foundTrimmedMessage, "No gen_ai.request.messages attribute found"); + }, +}; + +/** + * Check that trimming metadata is present and the truncation constraint is satisfied + * + * Validates: + * - gen_ai.input.messages.original_length (or gen_ai.request.messages.original_length) exists + * - The messages text content length is less than original_length (actual truncation occurred) + */ +export const checkTrimmingMetadata: Check = { + name: "checkTrimmingMetadata", + fn: (spans) => { + const aiSpans = extractGenAISpans(spans); + skipIf(aiSpans.length === 0, "No AI spans captured"); + + let foundMetadata = false; + + for (const span of aiSpans) { + // Check both new and legacy attribute names + const originalLength = + span.data?.["gen_ai.input.messages.original_length"] ?? + span.data?.["gen_ai.request.messages.original_length"]; + + if (originalLength === undefined) continue; + + foundMetadata = true; + + expect(originalLength, "original_length should be a number").to.be.a( + "number", + ); + expect( + originalLength, + "original_length should be greater than 0", + ).to.be.greaterThan(0); + + // Get the messages to validate content length constraint + const messagesRaw = + span.data?.["gen_ai.input.messages"] ?? + span.data?.["gen_ai.request.messages"]; + + if (messagesRaw !== undefined) { + // Get the string representation to measure actual content length + const messagesStr = + typeof messagesRaw === "string" + ? messagesRaw + : JSON.stringify(messagesRaw); + + expect( + messagesStr.length, + `Truncated messages content length (${messagesStr.length}) should be less than original_length (${originalLength})`, + ).to.be.lessThan(originalLength as number); + } + } + + expect( + foundMetadata, + "Should have at least one span with original_length metadata", + ).to.be.true; + }, +}; + +// ============================================================================= +// Agent-specific Checks +// ============================================================================= + +/** + * Check agent span hierarchy and gen_ai.agent.name propagation + * + * This check validates: + * 1. Agent spans (invoke_agent) exist and have gen_ai.agent.name + * 2. All child spans (ai_client, tool, handoff) inherit gen_ai.agent.name from their ancestor agent + * 3. No orphan gen_ai spans exist outside agent hierarchies + */ +export const checkAgentHierarchy: Check = { + name: "checkAgentHierarchy", + fn: (spans, config, testDef) => { + const aiSpans = extractGenAISpans(spans); + expect( + aiSpans.length, + "Should have at least one AI span", + ).to.be.greaterThan(0); + + // Build a map of span_id -> span for quick lookup (include all spans, not just gen_ai) + const spanMap = new Map(); + for (const span of spans) { + spanMap.set(span.span_id, span); + } + + // Find agent spans (invoke_agent pattern) + const agentSpans = aiSpans.filter( + (s) => + s.op?.match(/^gen_ai\.(invoke_agent|agent\.run|agent)$/) || + s.data?.["gen_ai.agent.name"] !== undefined, + ); + + expect( + agentSpans.length, + "Should have at least one agent span", + ).to.be.greaterThan(0); + + // For each agent span, verify it has gen_ai.agent.name + for (const agentSpan of agentSpans) { + const agentName = agentSpan.data?.["gen_ai.agent.name"]; + expect( + agentName, + `Agent span (${agentSpan.op}) should have gen_ai.agent.name attribute`, + ).to.exist; + } + + // Build set of agent span IDs for ancestry checking + const agentSpanIds = new Set(agentSpans.map((s) => s.span_id)); + + /** + * Find the ancestor agent span for a given span by walking up the parent chain + * Returns the agent span if found, undefined otherwise + */ + function findAncestorAgent(span: CapturedSpan): CapturedSpan | undefined { + let current: CapturedSpan | undefined = span; + const visited = new Set(); + + while (current) { + // Prevent infinite loops + if (visited.has(current.span_id)) { + break; + } + visited.add(current.span_id); + + // Check if current span is an agent span + if (agentSpanIds.has(current.span_id)) { + return current; + } + + // Move to parent + if (current.parent_span_id) { + current = spanMap.get(current.parent_span_id); + } else { + break; + } + } + + return undefined; + } + + // Categorize gen_ai spans by their relationship to agent spans + const childSpans: CapturedSpan[] = []; // Non-agent gen_ai spans that are descendants of agents + const orphanSpans: CapturedSpan[] = []; // gen_ai spans with no agent ancestor + + for (const span of aiSpans) { + // Skip agent spans themselves + if (agentSpanIds.has(span.span_id)) { + continue; + } + + const ancestorAgent = findAncestorAgent(span); + if (ancestorAgent) { + childSpans.push(span); + + // Verify gen_ai.agent.name matches the ancestor agent's name + const expectedAgentName = ancestorAgent.data?.["gen_ai.agent.name"]; + const actualAgentName = span.data?.["gen_ai.agent.name"]; + + expect( + actualAgentName, + `Child span (${span.op}, id: ${span.span_id.substring(0, 8)}) should have gen_ai.agent.name attribute`, + ).to.exist; + + expect( + actualAgentName, + `Child span (${span.op}) gen_ai.agent.name should match ancestor agent "${expectedAgentName}"`, + ).to.equal(expectedAgentName); + } else { + orphanSpans.push(span); + } + } + + // Fail if there are orphan gen_ai spans (not descended from any agent) + if (orphanSpans.length > 0) { + const orphanDetails = orphanSpans + .map((s) => `${s.op} (id: ${s.span_id.substring(0, 8)})`) + .join(", "); + throw new Error( + `Found ${orphanSpans.length} orphan gen_ai span(s) not descended from any agent span: ${orphanDetails}`, + ); + } + }, +}; diff --git a/src/test-cases/index.ts b/src/test-cases/index.ts new file mode 100644 index 0000000..4548220 --- /dev/null +++ b/src/test-cases/index.ts @@ -0,0 +1,66 @@ +/** + * Test Cases Index + * + * Centralized export of all test definitions + */ + +import { TestDefinition } from "../types.js"; +import { basicLLMTest } from "./llm/basic.js"; +import { multiTurnLLMTest } from "./llm/multi-turn.js"; +import { basicErrorLLMTest } from "./llm/basic-error.js"; +import { visionLLMTest } from "./llm/vision.js"; +import { longInputLLMTest } from "./llm/long-input.js"; +import { basicAgentTest } from "./agents/basic.js"; +import { toolCallAgentTest } from "./agents/tool-call.js"; +import { toolErrorAgentTest } from "./agents/tool-error.js"; +import { visionAgentTest } from "./agents/vision.js"; +import { longInputAgentTest } from "./agents/long-input.js"; + +/** + * All available test cases + */ +export const testCases = { + llm: { + basic: basicLLMTest, + multiTurn: multiTurnLLMTest, + basicError: basicErrorLLMTest, + vision: visionLLMTest, + longInput: longInputLLMTest, + }, + agents: { + basic: basicAgentTest, + toolCall: toolCallAgentTest, + toolError: toolErrorAgentTest, + vision: visionAgentTest, + longInput: longInputAgentTest, + }, +}; + +/** + * Get all LLM test cases + */ +export function getLLMTests(): TestDefinition[] { + return Object.values(testCases.llm); +} + +/** + * Get all agent test cases + */ +export function getAgentTests(): TestDefinition[] { + return Object.values(testCases.agents); +} + +/** + * Get all test cases + */ +export function getAllTests(): TestDefinition[] { + return [...getLLMTests(), ...getAgentTests()]; +} + +/** + * Get test case by name + */ +export function getTestByName(name: string): TestDefinition | undefined { + const allTests = getAllTests(); + return allTests.find((test) => test.name === name); +} diff --git a/src/test-cases/llm/basic-error.ts b/src/test-cases/llm/basic-error.ts new file mode 100644 index 0000000..906487c --- /dev/null +++ b/src/test-cases/llm/basic-error.ts @@ -0,0 +1,60 @@ +/** + * Basic Error LLM Test Case + * + * Tests that Sentry correctly captures API errors when the LLM call fails. + * Uses respx to mock a 500 Internal Server Error response. + */ + +import { expect } from "chai"; +import { TestDefinition, Check } from "../../types.js"; +import { checkAISpanCount } from "../checks.js"; +import { extractGenAISpans } from "../utils.js"; + +/** + * Check that the span has error information + */ +const checkErrorCaptured: Check = { + name: "checkErrorCaptured", + fn: (spans) => { + const aiSpans = extractGenAISpans(spans); + expect( + aiSpans.length, + "Should have at least one AI span", + ).to.be.greaterThan(0); + + // Find a span with error status or error data + const errorSpan = aiSpans.find( + (span) => + span.status === "internal_error" || + span.status === "unknown_error" || + span.data?.["error.type"] !== undefined || + span.data?.["http.status_code"] === 500, + ); + + expect(errorSpan, "Expected to find a span with error information").to + .exist; + }, +}; + +export const basicErrorLLMTest: TestDefinition = { + name: "Basic Error LLM Test", + description: "Tests error capture when API returns 500 Internal Server Error", + type: "llm", + + // This flag tells templates to mock an API error + causeAPIError: true, + + inputs: [ + { + model: "gpt-5-nano", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "What is the capital of France?" }, + ], + }, + ], + + checks: [checkAISpanCount({ min: 1 }), checkErrorCaptured], +}; + +export default basicErrorLLMTest; diff --git a/src/test-cases/llm/basic.ts b/src/test-cases/llm/basic.ts new file mode 100644 index 0000000..bbf728c --- /dev/null +++ b/src/test-cases/llm/basic.ts @@ -0,0 +1,43 @@ +/** + * Basic LLM Test Case + * + * Tests a simple completion call with system message and user prompt. + * Validates that Sentry captures the gen_ai span correctly. + */ + +import { TestDefinition } from "../../types.js"; +import { + checkAISpanCount, + checkChatSpanAttributes, + checkValidTokenUsage, + checkInputTokensCached, + checkOutputTokensReasoning, + checkInputMessagesSchema, +} from "../checks.js"; + +export const basicLLMTest: TestDefinition = { + name: "Basic LLM Test", + description: "Single completion call with system message", + type: "llm", + + inputs: [ + { + model: "gpt-5-nano", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "What is the capital of France?" }, + ], + }, + ], + + checks: [ + checkAISpanCount(1), + checkChatSpanAttributes, + checkValidTokenUsage, + checkInputMessagesSchema, + checkInputTokensCached, + checkOutputTokensReasoning, + ], +}; + +export default basicLLMTest; diff --git a/src/test-cases/llm/long-input.ts b/src/test-cases/llm/long-input.ts new file mode 100644 index 0000000..32c21f8 --- /dev/null +++ b/src/test-cases/llm/long-input.ts @@ -0,0 +1,59 @@ +/** + * Long Input LLM Test Case + * + * Tests that very long user messages (>20KB) trigger proper trimming + * of gen_ai.request.messages in the Sentry span data. + * + * Sentry SDKs trim long messages to prevent excessive span sizes. + * This test validates: + * 1. The message content is trimmed (less than original size) + * 2. Metadata about trimming is present (original_length) + * 3. Basic attributes are still captured correctly + * 4. Token counts reflect the actual (untrimmed) input + */ + +import { TestDefinition } from "../../types.js"; +import { + checkChatSpanAttributes, + checkMessageTrimming, + checkTrimmingMetadata, + checkInputMessagesSchema, +} from "../checks.js"; + +// Generate a long message that exceeds 20KB +// We'll repeat a pattern to create predictable content +const LONG_MESSAGE_PATTERN = + "This is a test message that will be repeated many times to create a very long input. "; +const REPETITIONS = 300; // ~25KB of text (85 chars * 300 = 25,500 bytes) +const LONG_MESSAGE = LONG_MESSAGE_PATTERN.repeat(REPETITIONS); + +export const longInputLLMTest: TestDefinition = { + name: "Long Input LLM Test", + description: "Tests message trimming for inputs > 20KB", + type: "llm", + + inputs: [ + { + model: "gpt-4o-mini", + messages: [ + { + role: "system", + content: "You are a helpful assistant. Respond briefly.", + }, + { + role: "user", + content: `Summarize this in one sentence: ${LONG_MESSAGE}`, + }, + ], + }, + ], + + checks: [ + checkChatSpanAttributes, + checkMessageTrimming, + checkTrimmingMetadata, + checkInputMessagesSchema, + ], +}; + +export default longInputLLMTest; diff --git a/src/test-cases/llm/multi-turn.ts b/src/test-cases/llm/multi-turn.ts new file mode 100644 index 0000000..a67806e --- /dev/null +++ b/src/test-cases/llm/multi-turn.ts @@ -0,0 +1,96 @@ +/** + * Multi-Turn LLM Test Case + * + * Tests a conversation with multiple back-and-forth exchanges. + * Validates that Sentry captures multiple gen_ai spans correctly. + */ + +import { expect } from "chai"; +import { TestDefinition, Check } from "../../types.js"; +import { + checkAISpanCount, + checkChatSpanAttributes, + checkValidTokenUsage, + checkInputTokensCached, + checkOutputTokensReasoning, + checkInputMessagesSchema, +} from "../checks.js"; +import { extractGenAISpans, skipIf } from "../utils.js"; + +/** + * Check that input tokens increase with each turn (more conversation history) + */ +const checkTokenProgression: Check = { + name: "checkTokenProgression", + fn: (spans) => { + const aiSpans = extractGenAISpans(spans); + skipIf( + aiSpans.length < 3, + `Expected 3 spans for multi-turn test, got ${aiSpans.length}`, + ); + + // Extract input token counts for each turn + const inputTokens = aiSpans.map( + (span) => span.data?.["gen_ai.usage.input_tokens"] as number, + ); + + // Input tokens should increase with each turn (more conversation history) + expect(inputTokens[1]).to.be.greaterThan(inputTokens[0]); + expect(inputTokens[2]).to.be.greaterThan(inputTokens[1]); + }, +}; + +export const multiTurnLLMTest: TestDefinition = { + name: "Multi-Turn LLM Test", + description: "Multi-turn conversation with back-and-forth exchanges", + type: "llm", + + inputs: [ + // Turn 1: Initial question + { + model: "gpt-5-nano", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "What is the capital of France?" }, + ], + }, + // Turn 2: Follow-up question + { + model: "gpt-5-nano", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "What is the capital of France?" }, + { role: "assistant", content: "The capital of France is Paris." }, + { role: "user", content: "What is the population of that city?" }, + ], + }, + // Turn 3: Another follow-up + { + model: "gpt-5-nano", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "What is the capital of France?" }, + { role: "assistant", content: "The capital of France is Paris." }, + { role: "user", content: "What is the population of that city?" }, + { + role: "assistant", + content: + "Paris has a population of approximately 2.2 million people in the city proper.", + }, + { role: "user", content: "What about the metropolitan area?" }, + ], + }, + ], + + checks: [ + checkAISpanCount(3), + checkChatSpanAttributes, + checkValidTokenUsage, + checkTokenProgression, + checkInputMessagesSchema, + checkInputTokensCached, + checkOutputTokensReasoning, + ], +}; + +export default multiTurnLLMTest; diff --git a/src/test-cases/llm/vision.ts b/src/test-cases/llm/vision.ts new file mode 100644 index 0000000..351991d --- /dev/null +++ b/src/test-cases/llm/vision.ts @@ -0,0 +1,60 @@ +/** + * Vision LLM Test Case + * + * Tests sending an image (base64 encoded PNG) to the LLM to verify + * multimodal input handling and Sentry span capture. + */ + +import { TestDefinition } from "../../types.js"; +import { + checkChatSpanAttributes, + checkValidTokenUsage, + checkInputMessagesSchema, + checkBinaryRedaction, +} from "../checks.js"; + +// Small 10x10 red PNG image encoded as base64 +const TEST_IMAGE_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8z8BQz0AEYBxVSF+FABJADveWkH6oAAAAAElFTkSuQmCC"; + +export const visionLLMTest: TestDefinition = { + name: "Vision LLM Test", + description: "Send an image to the LLM and ask about its contents", + type: "llm", + + inputs: [ + { + model: "gpt-4o-mini", + messages: [ + { + role: "system", + content: + "You are a helpful assistant that can analyze images. Be concise.", + }, + { + role: "user", + content: [ + { + type: "text", + text: "What color is this image? Reply with just the color name.", + }, + { + type: "image", + base64: TEST_IMAGE_BASE64, + mediaType: "image/png", + }, + ], + }, + ], + }, + ], + + checks: [ + checkChatSpanAttributes, + checkValidTokenUsage, + checkInputMessagesSchema, + checkBinaryRedaction, + ], +}; + +export default visionLLMTest; diff --git a/src/test-cases/utils.test.ts b/src/test-cases/utils.test.ts new file mode 100644 index 0000000..97be022 --- /dev/null +++ b/src/test-cases/utils.test.ts @@ -0,0 +1,154 @@ +/** + * Tests for assertAttributes utility + */ + +import { assertAttributes, AttributeSchema } from './utils.js'; +import { CapturedSpan } from '../types.js'; + +// Helper to create mock span +function createMockSpan(data: Record): CapturedSpan { + return { + span_id: 'test-span-id', + trace_id: 'test-trace-id', + op: 'gen_ai.chat', + description: 'Test span', + start_timestamp: 1234567890, + timestamp: 1234567891, + data, + }; +} + +console.log('Testing assertAttributes...\n'); + +// Test 1: true - attribute must exist +try { + const spans = [createMockSpan({ 'gen_ai.request.model': 'gpt-4' })]; + const schema: AttributeSchema = { 'gen_ai.request.model': true }; + assertAttributes(spans, schema); + console.log('✓ Test 1 passed: attribute exists'); +} catch (error) { + console.error('✗ Test 1 failed:', error instanceof Error ? error.message : error); +} + +// Test 2: false - attribute must NOT exist +try { + const spans = [createMockSpan({ 'other.attr': 'value' })]; + const schema: AttributeSchema = { 'gen_ai.request.model': false }; + assertAttributes(spans, schema); + console.log('✓ Test 2 passed: attribute does not exist'); +} catch (error) { + console.error('✗ Test 2 failed:', error instanceof Error ? error.message : error); +} + +// Test 3: Exact value match (string) +try { + const spans = [createMockSpan({ 'gen_ai.request.model': 'gpt-4' })]; + const schema: AttributeSchema = { 'gen_ai.request.model': 'gpt-4' }; + assertAttributes(spans, schema); + console.log('✓ Test 3 passed: exact string match'); +} catch (error) { + console.error('✗ Test 3 failed:', error instanceof Error ? error.message : error); +} + +// Test 4: Exact value match (number) +try { + const spans = [createMockSpan({ 'gen_ai.usage.input_tokens': 100 })]; + const schema: AttributeSchema = { 'gen_ai.usage.input_tokens': 100 }; + assertAttributes(spans, schema); + console.log('✓ Test 4 passed: exact number match'); +} catch (error) { + console.error('✗ Test 4 failed:', error instanceof Error ? error.message : error); +} + +// Test 5: Pattern match with wildcard +try { + const spans = [createMockSpan({ 'gen_ai.response.model': 'gpt-4-turbo-2024-01-01' })]; + const schema: AttributeSchema = { 'gen_ai.response.model': 'gpt-4*' }; + assertAttributes(spans, schema); + console.log('✓ Test 5 passed: wildcard pattern match'); +} catch (error) { + console.error('✗ Test 5 failed:', error instanceof Error ? error.message : error); +} + +// Test 6: Pattern match with multiple wildcards +try { + const spans = [createMockSpan({ 'gen_ai.response.model': 'gpt-4-turbo-preview' })]; + const schema: AttributeSchema = { 'gen_ai.response.model': 'gpt-*-*' }; + assertAttributes(spans, schema); + console.log('✓ Test 6 passed: multiple wildcards'); +} catch (error) { + console.error('✗ Test 6 failed:', error instanceof Error ? error.message : error); +} + +// Test 7: Should fail - attribute missing +try { + const spans = [createMockSpan({ 'other.attr': 'value' })]; + const schema: AttributeSchema = { 'gen_ai.request.model': true }; + assertAttributes(spans, schema); + console.error('✗ Test 7 failed: should have thrown error for missing attribute'); +} catch (error) { + console.log('✓ Test 7 passed: correctly detected missing attribute'); +} + +// Test 8: Should fail - attribute should not exist +try { + const spans = [createMockSpan({ 'gen_ai.request.model': 'gpt-4' })]; + const schema: AttributeSchema = { 'gen_ai.request.model': false }; + assertAttributes(spans, schema); + console.error('✗ Test 8 failed: should have thrown error for existing attribute'); +} catch (error) { + console.log('✓ Test 8 passed: correctly detected unwanted attribute'); +} + +// Test 9: Should fail - wrong value +try { + const spans = [createMockSpan({ 'gen_ai.request.model': 'gpt-4' })]; + const schema: AttributeSchema = { 'gen_ai.request.model': 'gpt-3.5' }; + assertAttributes(spans, schema); + console.error('✗ Test 9 failed: should have thrown error for wrong value'); +} catch (error) { + console.log('✓ Test 9 passed: correctly detected wrong value'); +} + +// Test 10: Should fail - pattern mismatch +try { + const spans = [createMockSpan({ 'gen_ai.response.model': 'claude-3-opus' })]; + const schema: AttributeSchema = { 'gen_ai.response.model': 'gpt-*' }; + assertAttributes(spans, schema); + console.error('✗ Test 10 failed: should have thrown error for pattern mismatch'); +} catch (error) { + console.log('✓ Test 10 passed: correctly detected pattern mismatch'); +} + +// Test 11: Multiple spans - all must match +try { + const spans = [ + createMockSpan({ 'gen_ai.request.model': 'gpt-4' }), + createMockSpan({ 'gen_ai.request.model': 'gpt-4' }), + ]; + const schema: AttributeSchema = { 'gen_ai.request.model': 'gpt-4' }; + assertAttributes(spans, schema); + console.log('✓ Test 11 passed: multiple spans all match'); +} catch (error) { + console.error('✗ Test 11 failed:', error instanceof Error ? error.message : error); +} + +// Test 12: Multiple attributes in schema +try { + const spans = [createMockSpan({ + 'gen_ai.request.model': 'gpt-4', + 'gen_ai.response.model': 'gpt-4-turbo-2024-01-01', + 'gen_ai.usage.input_tokens': 100, + })]; + const schema: AttributeSchema = { + 'gen_ai.request.model': 'gpt-4', + 'gen_ai.response.model': 'gpt-4*', + 'gen_ai.usage.input_tokens': true, + }; + assertAttributes(spans, schema); + console.log('✓ Test 12 passed: multiple attributes validated'); +} catch (error) { + console.error('✗ Test 12 failed:', error instanceof Error ? error.message : error); +} + +console.log('\nAll tests completed!'); diff --git a/src/test-cases/utils.ts b/src/test-cases/utils.ts new file mode 100644 index 0000000..16852ba --- /dev/null +++ b/src/test-cases/utils.ts @@ -0,0 +1,452 @@ +/** + * Common test utilities for span validation + */ + +import { expect } from "chai"; +import { CapturedSpan } from "../types.js"; +import { SkipCheckError } from "../validator.js"; + +/** + * Skip the current check with a reason + * @param reason - Why the check is being skipped + * @throws {SkipCheckError} + * @example + * if (!spans.length) { + * skip('No spans captured - cannot validate attributes'); + * } + */ +export function skip(reason: string): never { + throw new SkipCheckError(reason); +} + +/** + * Conditionally skip the current check + * @param condition - If true, skip the check + * @param reason - Why the check is being skipped + * @throws {SkipCheckError} + * @example + * skipIf(spans.length === 0, 'No spans captured'); + * skipIf(!config.supportsStreaming, 'Framework does not support streaming'); + */ +export function skipIf(condition: boolean, reason: string): void { + if (condition) { + throw new SkipCheckError(reason); + } +} + +/** + * Attribute schema for validation + * - true: attribute must exist + * - false: attribute must NOT exist + * - string with '*': must match pattern (glob-style) + * - string/number: must equal exact value + */ +export type AttributeSchema = { + [key: string]: boolean | string | number; +}; + +/** + * Extract AI spans from captured spans + * Only supports gen_ai.* prefixed operations + */ +export function extractGenAISpans(spans: CapturedSpan[]): CapturedSpan[] { + return spans.filter((s) => s.op && s.op.startsWith("gen_ai")); +} + +/** + * Check token usage attributes + */ +export interface TokenUsageChecks { + /** Check for input tokens */ + hasInputTokens?: boolean; + /** Check for output tokens */ + hasOutputTokens?: boolean; + /** Check for total tokens */ + hasTotalTokens?: boolean; + /** Minimum total tokens */ + minTotalTokens?: number; + /** Check that total = input + output */ + validateSum?: boolean; +} + +export function checkTokenUsage( + span: CapturedSpan, + checks: TokenUsageChecks = {}, +): void { + const { + hasInputTokens = true, + hasOutputTokens = true, + hasTotalTokens = true, + minTotalTokens, + validateSum = true, + } = checks; + + if (!span.data) { + throw new Error("Span has no data field"); + } + + // Extract token counts (only gen_ai.* prefix) + const inputTokens = span.data["gen_ai.usage.input_tokens"]; + const outputTokens = span.data["gen_ai.usage.output_tokens"]; + const totalTokens = span.data["gen_ai.usage.total_tokens"]; + + // Check presence + if (hasInputTokens) { + expect(inputTokens).to.exist; + expect(inputTokens).to.be.a("number"); + expect(inputTokens).to.be.greaterThan(0); + } + + if (hasOutputTokens) { + expect(outputTokens).to.exist; + expect(outputTokens).to.be.a("number"); + expect(outputTokens).to.be.greaterThan(0); + } + + if (hasTotalTokens) { + expect(totalTokens).to.exist; + expect(totalTokens).to.be.a("number"); + expect(totalTokens).to.be.greaterThan(0); + } + + // Check minimum total + if (minTotalTokens !== undefined && totalTokens) { + expect(totalTokens).to.be.at.least(minTotalTokens); + } + + // Validate sum + if (validateSum && inputTokens && outputTokens && totalTokens) { + expect(totalTokens).to.equal( + inputTokens + outputTokens, + "Total tokens should equal input + output tokens", + ); + } +} + +/** + * Check span hierarchy structure + */ +export interface SpanHierarchyChecks { + /** Expected parent operation pattern */ + parentOp?: RegExp; + /** Expected child operation pattern */ + childOp?: RegExp; + /** Minimum number of children */ + minChildren?: number; + /** Exact number of children */ + exactChildren?: number; +} + +export function checkSpanStructure( + spans: CapturedSpan[], + checks: SpanHierarchyChecks, +): void { + const { parentOp, childOp, minChildren, exactChildren } = checks; + + // Find parent span + const parentSpan = parentOp + ? spans.find((s) => s.op && s.op.match(parentOp)) + : undefined; + + if (parentOp && !parentSpan) { + throw new Error(`No parent span found matching pattern: ${parentOp}`); + } + + // Find child spans + let childSpans: CapturedSpan[] = []; + + if (parentSpan) { + // Find spans that reference this parent + childSpans = spans.filter((s) => s.parent_span_id === parentSpan.span_id); + } else if (childOp) { + // Just filter by operation pattern + childSpans = spans.filter((s) => s.op && s.op.match(childOp)); + } + + // Check child count + if (minChildren !== undefined) { + expect(childSpans.length).to.be.at.least( + minChildren, + `Should have at least ${minChildren} child span(s)`, + ); + } + + if (exactChildren !== undefined) { + expect(childSpans.length).to.equal( + exactChildren, + `Should have exactly ${exactChildren} child span(s)`, + ); + } + + // Validate child operations + if (childOp) { + childSpans.forEach((child, idx) => { + expect(child.op).to.match( + childOp, + `Child span ${idx} operation should match pattern`, + ); + }); + } +} + +/** + * Helper to print span summary for debugging + */ +export function printSpanSummary(spans: CapturedSpan[]): void { + console.log(`\n Captured ${spans.length} span(s):`); + spans.forEach((s, i) => { + const parent = s.parent_span_id + ? ` (parent: ${s.parent_span_id.substring(0, 8)})` + : ""; + console.log(` [${i}] ${s.op}${parent}`); + }); +} + +/** + * Match a value against a pattern (supports * wildcards) + */ +function matchPattern(value: string, pattern: string): boolean { + // Escape special regex characters except * + const regexPattern = pattern + .replace(/[.+?^${}()|[\]\\]/g, "\\$&") + .replace(/\*/g, ".*"); + + const regex = new RegExp(`^${regexPattern}$`); + return regex.test(value); +} + +/** + * Assert attributes on spans based on schema + * + * Schema format: + * - true: attribute must exist (any value) + * - false: attribute must NOT exist + * - string with '*': must match pattern (e.g., "gpt-4*" matches "gpt-4-turbo") + * - string/number: must equal exact value + * + * @param spans - List of spans to check (all spans must match schema) + * @param schema - Attribute schema to validate against + */ +export function assertAttributes( + spans: CapturedSpan[], + schema: AttributeSchema, +): void { + if (spans.length === 0) { + throw new Error("No spans provided to assertAttributes"); + } + + const errors: string[] = []; + + spans.forEach((span, spanIndex) => { + if (!span.data) { + errors.push(`Span ${spanIndex}: Missing data field`); + return; + } + + // Check each attribute in the schema + for (const [attrName, expected] of Object.entries(schema)) { + const actual = span.data[attrName]; + + if (expected === true) { + // Must exist + if (actual === undefined || actual === null) { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' must exist but is missing`, + ); + } + } else if (expected === false) { + // Must NOT exist + if (actual !== undefined && actual !== null) { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' must not exist but has value: ${actual}`, + ); + } + } else if (typeof expected === "string" && expected.includes("*")) { + // Pattern matching + if (actual === undefined || actual === null) { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' must exist for pattern matching but is missing`, + ); + } else if (typeof actual !== "string") { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' must be a string for pattern matching but is: ${typeof actual}`, + ); + } else if (!matchPattern(actual, expected)) { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' value '${actual}' does not match pattern '${expected}'`, + ); + } + } else { + // Exact value match + if (actual === undefined || actual === null) { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' must equal '${expected}' but is missing`, + ); + } else if (actual !== expected) { + errors.push( + `Span ${spanIndex}: Attribute '${attrName}' must equal '${expected}' but is '${actual}'`, + ); + } + } + } + }); + + if (errors.length > 0) { + throw new Error(`Attribute validation failed:\n ${errors.join("\n ")}`); + } +} + +// ============================================================================= +// Span Type Filtering Helpers +// ============================================================================= + +/** + * Find invoke_agent spans (top-level agent invocation) + */ +export function findAgentSpans(spans: CapturedSpan[]): CapturedSpan[] { + return spans.filter( + (s) => + s.op?.match(/^gen_ai\.(invoke_agent|agent\.run|agent)$/) || + (s.data?.["gen_ai.agent.name"] !== undefined && + s.op?.match(/^gen_ai\.invoke_agent/)), + ); +} + +/** + * Find ai_client/chat spans (LLM API calls) + */ +export function findChatSpans(spans: CapturedSpan[]): CapturedSpan[] { + return spans.filter((s) => + s.op?.match(/^gen_ai\.(chat|completion|generate)/), + ); +} + +/** + * Find tool execution spans + * Matches spans with operations like gen_ai.execute_tool, gen_ai.tool, etc. + */ +export function findToolSpans(spans: CapturedSpan[]): CapturedSpan[] { + return spans.filter( + (s) => + s.op?.match(/^gen_ai\.(execute_tool|tool|tool_call)/) || + s.data?.["gen_ai.tool.name"] !== undefined, + ); +} + +/** + * Find handoff spans (agent-to-agent handoffs) + */ +export function findHandoffSpans(spans: CapturedSpan[]): CapturedSpan[] { + return spans.filter((s) => s.op?.match(/^gen_ai\.handoff/)); +} + +/** + * Schema for tool input validation + * - true: argument must exist (any value) + * - false: argument must NOT exist + * - string/number: argument must equal exact value + */ +export type ToolInputSchema = { + [key: string]: boolean | string | number; +}; + +/** + * Assert that a tool span has the expected input arguments + * + * Tool input is typically stored in gen_ai.tool.input as a JSON string + * + * @param span - The tool span to check + * @param schema - Expected arguments schema + */ +export function assertToolInput( + span: CapturedSpan, + schema: ToolInputSchema, +): void { + const toolInput = span.data?.["gen_ai.tool.input"]; + + if (toolInput === undefined) { + throw new Error(`Tool span is missing gen_ai.tool.input attribute`); + } + + // Parse the tool input (it's usually a JSON string) + let parsedInput: Record; + if (typeof toolInput === "string") { + try { + parsedInput = JSON.parse(toolInput); + } catch { + throw new Error(`Tool input is not valid JSON: ${toolInput}`); + } + } else if (typeof toolInput === "object" && toolInput !== null) { + parsedInput = toolInput as Record; + } else { + throw new Error(`Unexpected tool input type: ${typeof toolInput}`); + } + + const errors: string[] = []; + + for (const [argName, expected] of Object.entries(schema)) { + const actual = parsedInput[argName]; + + if (expected === true) { + // Must exist + if (actual === undefined) { + errors.push(`Tool argument '${argName}' must exist but is missing`); + } + } else if (expected === false) { + // Must NOT exist + if (actual !== undefined) { + errors.push( + `Tool argument '${argName}' must not exist but has value: ${actual}`, + ); + } + } else { + // Exact value match (convert to string for comparison since JSON parsing may vary) + if (actual === undefined) { + errors.push( + `Tool argument '${argName}' must equal '${expected}' but is missing`, + ); + } else { + // Compare as strings to handle type coercion (e.g., "4" vs 4) + const actualStr = String(actual); + const expectedStr = String(expected); + if (actualStr !== expectedStr) { + errors.push( + `Tool argument '${argName}' must equal '${expected}' but is '${actual}'`, + ); + } + } + } + } + + if (errors.length > 0) { + throw new Error(`Tool input validation failed:\n ${errors.join("\n ")}`); + } +} + +/** + * Get tool input arguments from a tool span + * Returns the parsed arguments object, or undefined if not available + */ +export function getToolInput( + span: CapturedSpan, +): Record | undefined { + const toolInput = span.data?.["gen_ai.tool.input"]; + + if (toolInput === undefined) { + return undefined; + } + + if (typeof toolInput === "string") { + try { + return JSON.parse(toolInput); + } catch { + return undefined; + } + } + + if (typeof toolInput === "object" && toolInput !== null) { + return toolInput as Record; + } + + return undefined; +} diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..bc1943f --- /dev/null +++ b/src/types.ts @@ -0,0 +1,168 @@ +/** + * Core type definitions for the test orchestrator + */ + +/** + * Check function signature + * @param spans - Captured spans from the test run + * @param config - Framework configuration + * @param testDef - The test definition being run + */ +export type CheckFunction = ( + spans: CapturedSpan[], + config: FrameworkConfig, + testDef: TestDefinition, +) => void | Promise; + +/** + * Check definition with name and function + */ +export interface Check { + name: string; + fn: CheckFunction; +} + +export interface TestDefinition { + name: string; + description: string; + /** Test type: determines which frameworks this test can run on */ + type: "llm" | "agent"; + agent?: AgentDefinition; + inputs: TestInput[]; + /** If true, the test should intentionally cause an API error (e.g., invalid model name) */ + causeAPIError?: boolean; + /** Array of check functions to run */ + checks: Check[]; +} + +export interface AgentDefinition { + name: string; + description: string; + tools: ToolDefinition[]; +} + +export interface ToolDefinition { + name: string; + description: string; + parameters: Record; + result?: any; + error?: string; +} + +/** Text content part for multimodal messages */ +export interface TextContentPart { + type: "text"; + text: string; +} + +/** Image content part for multimodal messages */ +export interface ImageContentPart { + type: "image"; + /** Base64 encoded image data (without data URI prefix) */ + base64: string; + /** MIME type of the image (e.g., "image/png", "image/jpeg") */ + mediaType: string; +} + +/** Content can be a simple string or an array of content parts for multimodal */ +export type MessageContent = string | (TextContentPart | ImageContentPart)[]; + +export interface Message { + role: "system" | "user" | "assistant" | "tool"; + content: MessageContent; + name?: string; + tool_call_id?: string; +} + +export interface TestInput { + model: string; + messages: Message[]; + [key: string]: any; +} + +export interface CapturedSpan { + span_id: string; + trace_id: string; + op: string; + description?: string; + start_timestamp: number; + timestamp: number; + data?: Record; + tags?: Record; + [key: string]: any; +} + +export interface FrameworkConfig { + name: string; + platform: "js" | "py"; + type: "llm-only" | "agentic"; + version: string; + sentryVersion: string; + // Optional: Path to template file (set when using discovered frameworks) + templatePath?: string; + category?: string; + dependencies?: Array<{ package: string; version: string }>; + // Python only: execution mode for the framework + executionMode?: "sync" | "async" | "both"; + // Streaming mode: whether the framework supports streaming responses + streamingMode?: "streaming" | "blocking" | "both"; + // Model overrides: Some frameworks use different models than requested + modelOverrides?: { + request?: string; + response?: string; + }; + // Skip configuration: Tests or checks that should be skipped + skip?: { + tests?: string[]; // Array of test names to skip entirely + checks?: { + // Per-test check skipping + [testName: string]: string[]; // Array of check method names to skip + }; + }; +} + +export interface CheckResult { + name: string; + status: "passed" | "failed" | "skipped"; + error?: string; + skipReason?: string; +} + +export interface TestRun { + id: string; + /** Original index in the test matrix, used for consistent ordering in reports */ + index?: number; + framework: FrameworkConfig; + testDefinition: TestDefinition; + status: "pending" | "running" | "passed" | "failed" | "error" | "skipped"; + startTime?: number; + endTime?: number; + error?: string; + spans?: CapturedSpan[]; + checkResults?: CheckResult[]; + skipReason?: string; +} + +export interface TestReport { + totalTests: number; + passed: number; + failed: number; + errors: number; + skipped: number; + duration: number; + runs: TestRun[]; +} + +export interface RunnerContext { + runId: string; + framework: FrameworkConfig; + testDefinition: TestDefinition; + sentryDsn: string; + workDir: string; + // Python only: if true, render async version; if false, render sync version + isAsync?: boolean; + // If true, render streaming version; if false, render non-streaming version + isStreaming?: boolean; + // Controls whether to print verbose console output (default: true) + verbose?: boolean; +} diff --git a/src/validator.ts b/src/validator.ts new file mode 100644 index 0000000..ade172d --- /dev/null +++ b/src/validator.ts @@ -0,0 +1,146 @@ +/** + * Validator - runs test assertions on captured spans + */ + +import { + CapturedSpan, + TestDefinition, + FrameworkConfig, + CheckResult, + Check, +} from "./types.js"; + +/** + * Custom error that carries check results + */ +export class ValidationError extends Error { + constructor( + message: string, + public checkResults: CheckResult[], + ) { + super(message); + this.name = "ValidationError"; + } +} + +/** + * Custom error for skipping checks dynamically + * Thrown by skip() and skipIf() helpers in test utilities + */ +export class SkipCheckError extends Error { + constructor(public reason: string) { + super(reason); + this.name = "SkipCheckError"; + } +} + +export class Validator { + private verbose: boolean = false; + + setVerbose(verbose: boolean): void { + this.verbose = verbose; + } + + /** + * Run validation checks on captured spans + * Uses the checks array from the test definition + * Returns check results for detailed reporting + */ + async validate( + spans: CapturedSpan[], + testDefinition: TestDefinition, + frameworkConfig: FrameworkConfig, + onCheckStart?: (checkName: string) => void, + onCheckResult?: (result: CheckResult) => void, + ): Promise { + const checkResults: CheckResult[] = []; + const errors: Error[] = []; + + // Get checks array from test definition + const checks: Check[] = testDefinition.checks || []; + + if (checks.length === 0) { + if (this.verbose) { + console.log(" No checks defined for this test"); + } + return checkResults; + } + + // Get test name for skip lookup + const testName = testDefinition.name; + const skippedChecks = frameworkConfig.skip?.checks?.[testName] || []; + + // Run all checks + for (const check of checks) { + const checkName = check.name; + + // Notify that check is starting + onCheckStart?.(checkName); + + // Check if this check is skipped for this framework + if (skippedChecks.includes(checkName)) { + const result: CheckResult = { + name: checkName, + status: "skipped", + skipReason: "Not supported by this framework", + }; + checkResults.push(result); + onCheckResult?.(result); + if (this.verbose) { + console.log(` ⊘ ${checkName} skipped (not supported)`); + } + continue; + } + + try { + await check.fn(spans, frameworkConfig, testDefinition); + const result: CheckResult = { name: checkName, status: "passed" }; + checkResults.push(result); + onCheckResult?.(result); + if (this.verbose) { + console.log(` ✓ ${checkName} passed`); + } + } catch (error) { + // Handle dynamic skip from within the check + if (error instanceof SkipCheckError) { + const result: CheckResult = { + name: checkName, + status: "skipped", + skipReason: error.reason, + }; + checkResults.push(result); + onCheckResult?.(result); + if (this.verbose) { + console.log(` ⊘ ${checkName} skipped: ${error.reason}`); + } + continue; + } + + // Handle regular failures + const errorMsg = error instanceof Error ? error.message : String(error); + const result: CheckResult = { + name: checkName, + status: "failed", + error: errorMsg, + }; + checkResults.push(result); + onCheckResult?.(result); + if (this.verbose) { + console.error(` ✗ ${checkName} failed: ${errorMsg}`); + } + errors.push(error instanceof Error ? error : new Error(errorMsg)); + } + } + + // If any check failed, throw combined error with check results + if (errors.length > 0) { + const errorMessages = errors.map((e) => e.message).join("\n"); + throw new ValidationError( + `${errors.length} check(s) failed:\n${errorMessages}`, + checkResults, + ); + } + + return checkResults; + } +} diff --git a/shared/orchestration/tsconfig.json b/tsconfig.json similarity index 76% rename from shared/orchestration/tsconfig.json rename to tsconfig.json index 4657128..4eb416d 100644 --- a/shared/orchestration/tsconfig.json +++ b/tsconfig.json @@ -1,21 +1,20 @@ { "compilerOptions": { "target": "ES2022", - "module": "ESNext", + "module": "ES2022", "lib": ["ES2022"], - "moduleResolution": "bundler", - "resolveJsonModule": true, - "allowJs": true, + "moduleResolution": "node", "outDir": "./dist", "rootDir": "./src", "strict": true, "esModuleInterop": true, "skipLibCheck": true, "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, "declaration": true, "declarationMap": true, "sourceMap": true }, "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] + "exclude": ["node_modules", "dist", "archive"] }