From 78568e340a8eb4b6ea357bb87a4a088bf447f9eb Mon Sep 17 00:00:00 2001 From: Louis Choquel Date: Wed, 3 Sep 2025 11:05:15 +0200 Subject: [PATCH 01/10] docs/README-Add-link-to-VS-Code-extension (#35) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 329c85b..1b7b9d9 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,10 @@ cocode swe-ai-instruction-update v1.0.0 path/to/your/local/repository The results of these commands will be saved in a `results` (default behavior) folder at the root of your project. +### IDE extension + +We **highly** recommend installing our own extension for PLX files into your IDE of choice. You can find it in the [VS Code Marketplace](https://marketplace.visualstudio.com/items?itemName=Pipelex.pipelex). + ## 🔧 Other Features ### 🤖 **AI-Powered Software Engineering Analysis** From b27f457562a73a67e938613b64216dca8f6aadc9 Mon Sep 17 00:00:00 2001 From: Thomas Hebrard Date: Wed, 3 Sep 2025 11:07:55 +0200 Subject: [PATCH 02/10] Fix/readme (#36) * fix readme * bump version --- CHANGELOG.md | 7 +++++++ README.md | 4 ++-- pipelex.toml | 4 ++-- pyproject.toml | 2 +- uv.lock | 2 +- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42b7c4e..42a7bbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [v0.1.2] - 2025-09-03 + +### Changed + +- Deactivates pipeline tracking and activity tracking in `pipelex.toml`. +- Added IDE extension link to `README.md`. + ## [v0.1.1] - 2025-09-02 ### Added diff --git a/README.md b/README.md index 1b7b9d9..64d7bf3 100644 --- a/README.md +++ b/README.md @@ -43,10 +43,10 @@ Some complex pipelines require GCP credentials (See [GCP credentials](https://do cocode swe-doc-update v1.0.0 path/to/your/local/repository # Proofread documentation against codebase -cocode swe-doc-proofread --doc-dir docs path/to/your/local/repository +cocode swe-doc-proofread --doc-dir docs path/to/your/local/repository # Requires gemini access (gcp credentials file) # Generate changelog from version diff -cocode swe-from-repo-diff write_changelog v1.0.0 path/to/your/local/repository +cocode swe-from-repo-diff write_changelog v1.0.0 path/to/your/local/repository # Requires Anthropic API key # Update AI instructions (AGENTS.md, CLAUDE.md, cursor rules) based on code changes cocode swe-ai-instruction-update v1.0.0 path/to/your/local/repository diff --git a/pipelex.toml b/pipelex.toml index 0081cbe..52ef3f2 100644 --- a/pipelex.toml +++ b/pipelex.toml @@ -19,7 +19,7 @@ page_output_text_file_name = "page_text.md" [pipelex.feature_config] # WIP/Experimental feature flags -is_pipeline_tracking_enabled = true -is_activity_tracking_enabled = true +is_pipeline_tracking_enabled = false +is_activity_tracking_enabled = false is_reporting_enabled = true diff --git a/pyproject.toml b/pyproject.toml index 22e92b9..7e1ac76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "cocode" -version = "0.1.1" +version = "0.1.2" description = "Cocode is the friend of your code" authors = [{ name = "Evotis S.A.S.", email = "evotis@pipelex.com" }] maintainers = [{ name = "Pipelex staff", email = "oss@pipelex.com" }] diff --git a/uv.lock b/uv.lock index 0208936..0d636d0 100644 --- a/uv.lock +++ b/uv.lock @@ -483,7 +483,7 @@ wheels = [ [[package]] name = "cocode" -version = "0.1.1" +version = "0.1.2" source = { editable = "." } dependencies = [ { name = "pipelex", extra = ["anthropic", "bedrock", "google"] }, From a6ce0121f4437ea3083f8f000222dbf68a5cbac9 Mon Sep 17 00:00:00 2001 From: Louis Choquel Date: Fri, 5 Sep 2025 15:00:01 +0200 Subject: [PATCH 03/10] Feature/docs fixes and cleanups (#40) * Add docs for github commands * swe_from_file_cmd use PipeCode the same way as swe_from_repo_cmd added some pipe codes to enum * Renamed write_changelog_v2 to write_changelog_enhanced * Rename check_docs_consistency to check_doc_inconsistencies --- CHANGELOG.md | 2 +- CLI_README.md | 98 +++++++++++++++++++ README.md | 33 +++++++ cocode/cli.py | 27 ++++- .../pipelines/swe_diff/swe_diff.plx | 2 +- .../pipelines/swe_docs/swe_docs.plx | 2 +- docs/pages/commands.md | 78 +++++++++++++++ docs/pages/examples.md | 52 ++++++++++ 8 files changed, 289 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7633295..112f73f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,7 +45,7 @@ ## [v0.0.11] - 2025-08-02 ### Added - - Added comprehensive changelog generation pipeline (`write_changelog_v2`) with three-stage processing: draft generation, polishing, and markdown formatting + - Added comprehensive changelog generation pipeline (`write_changelog_enhanced`) with three-stage processing: draft generation, polishing, and markdown formatting - Added `DraftChangelog` concept definition for intermediate changelog processing - Added AI-powered `draft_changelog_from_git_diff` pipe that analyzes code diffs using LLM to extract changes, improvements, and features - Added `polish_changelog` pipe that removes redundancy, groups related changes, and applies markdown formatting diff --git a/CLI_README.md b/CLI_README.md index 9d94fe2..83436f6 100644 --- a/CLI_README.md +++ b/CLI_README.md @@ -69,6 +69,13 @@ cocode swe-from-file extract_features_recap ./results/pipelex-docs.txt \ # SWE analysis: Generate changelog from git diff cocode swe-from-repo-diff write_changelog v0.2.4 ../pipelex-cookbook/ \ --output-filename "changelog.md" + +# GitHub operations +cocode github auth # Check authentication status +cocode github repo-info pipelex/cocode # Get repository information +cocode github check-branch pipelex/cocode main # Check if branch exists +cocode github list-branches pipelex/cocode --limit 10 # List branches +cocode github sync-labels pipelex/cocode ./labels.json # Sync labels ``` ## Overview @@ -242,6 +249,84 @@ The tool focuses on critical issues that would break user code, such as: - Wrong import paths - Critical type mismatches +### `github` - GitHub Repository Management + +Manage GitHub repositories, branches, and labels. + +**Subcommands:** + +#### `github auth` +Check GitHub authentication status and display rate limit information. + +```bash +cocode github auth +``` + +#### `github repo-info` +Get detailed information about a GitHub repository. + +```bash +# Using owner/repo format +cocode github repo-info pipelex/cocode + +# Using repository ID +cocode github repo-info 123456789 +``` + +#### `github check-branch` +Check if a specific branch exists in a repository. + +```bash +cocode github check-branch pipelex/cocode main +cocode github check-branch pipelex/cocode feature-branch +``` + +#### `github list-branches` +List branches in a repository with optional limit. + +```bash +# List first 10 branches (default) +cocode github list-branches pipelex/cocode + +# List first 20 branches +cocode github list-branches pipelex/cocode --limit 20 +``` + +#### `github sync-labels` +Synchronize issue labels from a JSON file to a repository. + +```bash +# Dry run to preview changes +cocode github sync-labels pipelex/cocode ./labels.json --dry-run + +# Sync labels, keeping existing ones +cocode github sync-labels pipelex/cocode ./labels.json + +# Sync labels and delete extras not in JSON file +cocode github sync-labels pipelex/cocode ./labels.json --delete-extra +``` + +**Label JSON Format:** +```json +[ + { + "name": "bug", + "color": "d73a4a", + "description": "Something isn't working" + }, + { + "name": "enhancement", + "color": "a2eeef", + "description": "New feature or request" + } +] +``` + +**Options:** +- `--dry-run`: Preview changes without making them +- `--delete-extra`: Remove labels not in the JSON file +- `--limit`: Maximum number of items to display (for list commands) + ### `validate` - Configuration Validation Validate setup and pipelines. @@ -313,6 +398,19 @@ cocode repox --path-pattern ".cursor/rules" --include-pattern "*.mdc" - Use `cocode validate` to check configuration and pipelines - Configuration files are loaded from `pipelex_libraries/` directory +### GitHub Authentication + +For GitHub commands, authentication is handled via the `GITHUB_TOKEN` environment variable or PyGithub's default authentication methods: + +1. **Environment Variable**: Set `GITHUB_TOKEN` in your `.env` file or environment +2. **GitHub CLI**: If you have `gh` CLI installed and authenticated +3. **Personal Access Token**: Create at https://github.com/settings/tokens + +```bash +# Check authentication status +cocode github auth +``` + ## Getting Started 1. **Validate Setup**: Run `cocode validate` to ensure everything is configured correctly diff --git a/README.md b/README.md index d8d6191..900b3ea 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,13 @@ cocode swe-from-repo-diff write_changelog v1.0.0 path/to/your/local/repository # # Update AI instructions (AGENTS.md, CLAUDE.md, cursor rules) based on code changes cocode swe-ai-instruction-update v1.0.0 path/to/your/local/repository + +# GitHub operations +cocode github auth # Check GitHub authentication status +cocode github repo-info owner/repo # Get repository information +cocode github check-branch owner/repo main # Check if branch exists +cocode github list-branches owner/repo # List repository branches +cocode github sync-labels owner/repo labels.json # Sync labels from JSON file ``` ### 📁 Output Location @@ -89,6 +96,32 @@ Choose the right format for your needs: - **Tree**: Directory structure visualization - **Import List**: Dependency analysis format +### 🔗 **GitHub Integration** +Powerful GitHub repository management features: +- **Authentication**: Check and manage GitHub authentication status +- **Repository Info**: Get detailed information about repositories +- **Branch Management**: Check branches, list branches +- **Label Sync**: Synchronize issue labels across repositories from JSON templates + +### GitHub Commands +```bash +# Check authentication status +cocode github auth + +# Get repository information +cocode github repo-info pipelex/cocode + +# Check if a branch exists +cocode github check-branch pipelex/cocode feature-branch + +# List branches (with limit) +cocode github list-branches pipelex/cocode --limit 20 + +# Sync labels from JSON file +cocode github sync-labels pipelex/cocode ./labels.json --dry-run +cocode github sync-labels pipelex/cocode ./labels.json --delete-extra +``` + ### Commands for Other Features ## ⚠️ Limitations diff --git a/cocode/cli.py b/cocode/cli.py index 5435af2..90a0333 100644 --- a/cocode/cli.py +++ b/cocode/cli.py @@ -40,20 +40,43 @@ class PipeCode(StrEnum): EXTRACT_ENVIRONMENT_BUILD = "extract_environment_build" EXTRACT_CODING_STANDARDS = "extract_coding_standards" EXTRACT_TEST_STRATEGY = "extract_test_strategy" + EXTRACT_CONTEXTUAL_GUIDELINES = "extract_contextual_guidelines" EXTRACT_COLLABORATION = "extract_collaboration" + EXTRACT_FEATURES_RECAP = "extract_features_recap" DOC_PROOFREAD = "doc_proofread" DOC_UPDATE = "doc_update" AI_INSTRUCTION_UPDATE = "ai_instruction_update" + # SWE diff analysis + WRITE_CHANGELOG = "write_changelog" + WRITE_CHANGELOG_ENHANCED = "write_changelog_enhanced" + + # Text utilities + GENERATE_SPLIT_IDENTIFIERS = "generate_split_identifiers" + + # SWE docs consistency check + CHECK_DOCS_INCONSISTENCIES = "check_doc_inconsistencies" + def _get_pipe_descriptions() -> str: """Generate help text with pipe descriptions from TOML.""" descriptions = { "extract_onboarding_documentation": "Extract comprehensive onboarding documentation from software project docs", + "extract_fundamentals": "Extract fundamental project information from documentation", + "extract_environment_build": "Extract environment setup and build information from documentation", + "extract_coding_standards": "Extract code quality and style information from documentation", + "extract_test_strategy": "Extract testing strategy and procedures from documentation", + "extract_contextual_guidelines": "Extract contextual development guidelines from documentation", + "extract_collaboration": "Extract collaboration and workflow information from documentation", + "extract_features_recap": "Extract and analyze software features from documentation", "doc_proofread": "Systematically proofread documentation against actual codebase to find inconsistencies", "doc_update": "Generate documentation update suggestions for docs/ directory", "ai_instruction_update": "Generate AI instruction update suggestions for AGENTS.md, CLAUDE.md, cursor rules", + "write_changelog": "Write a comprehensive changelog for a software project from git diff", + "write_changelog_enhanced": "Write a comprehensive changelog with draft and polish steps from git diff", + "generate_split_identifiers": "Analyze large text and generate optimal split identifiers", + "check_doc_inconsistencies": "Identify inconsistencies in a set of software engineering documents", } help_text = "\n\n" @@ -256,8 +279,8 @@ def swe_from_repo_cmd( @app.command("swe-from-file") def swe_from_file_cmd( pipe_code: Annotated[ - str, - typer.Argument(help="Pipeline code to execute for SWE analysis"), + PipeCode, + typer.Argument(help=f"Pipeline code to execute for SWE analysis.\n\n{_get_pipe_descriptions()}"), ], input_file_path: Annotated[ str, diff --git a/cocode/pipelex_libraries/pipelines/swe_diff/swe_diff.plx b/cocode/pipelex_libraries/pipelines/swe_diff/swe_diff.plx index 1e335e3..a9152d2 100644 --- a/cocode/pipelex_libraries/pipelines/swe_diff/swe_diff.plx +++ b/cocode/pipelex_libraries/pipelines/swe_diff/swe_diff.plx @@ -18,7 +18,7 @@ steps = [ { pipe = "format_changelog_as_markdown", result = "markdown_changelog" }, ] -[pipe.write_changelog_v2] +[pipe.write_changelog_enhanced] type = "PipeSequence" definition = "Write a comprehensive changelog for a software project" inputs = { git_diff = "GitDiff" } diff --git a/cocode/pipelex_libraries/pipelines/swe_docs/swe_docs.plx b/cocode/pipelex_libraries/pipelines/swe_docs/swe_docs.plx index 12120d3..3b947ea 100644 --- a/cocode/pipelex_libraries/pipelines/swe_docs/swe_docs.plx +++ b/cocode/pipelex_libraries/pipelines/swe_docs/swe_docs.plx @@ -14,7 +14,7 @@ CollaborationDoc = "A comprehensive overview of the collaboration and workflow i OnboardingDocumentation = "Complete set of documentation needed for onboarding new developers to a project." [pipe] -[pipe.check_docs_consistency] +[pipe.check_doc_inconsistencies] type = "PipeLLM" definition = "Identify inconsistencies in a set of software engineering documents." inputs = { repo_text = "SoftwareDoc" } diff --git a/docs/pages/commands.md b/docs/pages/commands.md index b04617c..37c04f2 100644 --- a/docs/pages/commands.md +++ b/docs/pages/commands.md @@ -98,6 +98,84 @@ Detect critical inconsistencies between documentation and actual codebase that c - `--dry` - Dry run without API calls - Plus all filtering options from `repox` +## github + +GitHub repository management commands. + +### github auth + +Check GitHub authentication status. + +```bash +cocode github auth +``` + +Displays authenticated user info and API rate limits. + +### github repo-info + +Get repository information. + +```bash +cocode github repo-info REPO +``` + +**Arguments:** +- `REPO` - Repository as `owner/repo` or repository ID + +### github check-branch + +Check if a branch exists. + +```bash +cocode github check-branch REPO BRANCH +``` + +**Arguments:** +- `REPO` - Repository as `owner/repo` or repository ID +- `BRANCH` - Branch name to check + +### github list-branches + +List repository branches. + +```bash +cocode github list-branches [OPTIONS] REPO +``` + +**Arguments:** +- `REPO` - Repository as `owner/repo` or repository ID + +**Options:** +- `--limit` - Maximum branches to show (default: 10) + +### github sync-labels + +Sync issue labels from JSON file. + +```bash +cocode github sync-labels [OPTIONS] REPO LABELS_FILE +``` + +**Arguments:** +- `REPO` - Repository as `owner/repo` or repository ID +- `LABELS_FILE` - JSON file with label definitions + +**Options:** +- `--dry-run` - Preview changes without applying +- `--delete-extra` - Remove labels not in JSON file + +**Label JSON format:** +```json +[ + { + "name": "bug", + "color": "d73a4a", + "description": "Something isn't working" + } +] +``` + ## swe-doc-update Update documentation based on code changes. diff --git a/docs/pages/examples.md b/docs/pages/examples.md index 16e7d48..f1bf47a 100644 --- a/docs/pages/examples.md +++ b/docs/pages/examples.md @@ -76,6 +76,58 @@ cocode swe-from-file extract_features_recap results/docs.txt \ --output-filename features.md ``` +## GitHub operations + +### Repository management + +```bash +# Check authentication +cocode github auth + +# Get repo details +cocode github repo-info pipelex/cocode + +# Check if feature branch exists +cocode github check-branch pipelex/cocode feature/new-feature + +# List all branches +cocode github list-branches pipelex/cocode --limit 20 +``` + +### Label synchronization + +```bash +# Preview label changes +cocode github sync-labels pipelex/cocode ./labels.json --dry-run + +# Apply labels (keep existing) +cocode github sync-labels pipelex/cocode ./labels.json + +# Full sync (remove extras) +cocode github sync-labels pipelex/cocode ./labels.json --delete-extra +``` + +**Example labels.json:** +```json +[ + { + "name": "bug", + "color": "d73a4a", + "description": "Something isn't working" + }, + { + "name": "enhancement", + "color": "a2eeef", + "description": "New feature or request" + }, + { + "name": "documentation", + "color": "0075ca", + "description": "Improvements or additions to documentation" + } +] +``` + ## Common workflows ### Full project analysis From 0345545414dad282fd5dba0739aa2872609a26a0 Mon Sep 17 00:00:00 2001 From: Thomas Hebrard Date: Fri, 5 Sep 2025 23:40:21 +0200 Subject: [PATCH 04/10] fix/cursor_rules (#41) * fix/cursor_rules * fix rules --- .cursor/rules/base_models.mdc | 14 - .cursor/rules/check-pipes.mdc | 13 - .cursor/rules/coding_standards.mdc | 124 ++++ .cursor/rules/docs.mdc | 8 + .cursor/rules/llms.mdc | 30 +- .cursor/rules/pipelex.mdc | 28 +- .cursor/rules/pipes.mdc | 224 ------- .cursor/rules/piping.mdc | 21 - .cursor/rules/prompt-templates.mdc | 62 -- .cursor/rules/pytest.mdc | 27 +- .cursor/rules/structures.mdc | 63 -- .cursor/rules/tdd.mdc | 28 + .github/copilot-instructions.md | 926 +++++++++++++++++++++++++++++ AGENTS.md | 909 +++++++++++++++++++++++++++- CLAUDE.md | 896 +++++++++++++++++++++++++++- CLI_README.md | 4 +- docs/pages/commands.md | 13 - 17 files changed, 2890 insertions(+), 500 deletions(-) delete mode 100644 .cursor/rules/base_models.mdc delete mode 100644 .cursor/rules/check-pipes.mdc create mode 100644 .cursor/rules/coding_standards.mdc create mode 100644 .cursor/rules/docs.mdc delete mode 100644 .cursor/rules/pipes.mdc delete mode 100644 .cursor/rules/piping.mdc delete mode 100644 .cursor/rules/prompt-templates.mdc delete mode 100644 .cursor/rules/structures.mdc create mode 100644 .cursor/rules/tdd.mdc create mode 100644 .github/copilot-instructions.md diff --git a/.cursor/rules/base_models.mdc b/.cursor/rules/base_models.mdc deleted file mode 100644 index 777b154..0000000 --- a/.cursor/rules/base_models.mdc +++ /dev/null @@ -1,14 +0,0 @@ ---- -description: -globs: -alwaysApply: true ---- -Rule to create BaseModels: - -- Respect Pydantic v2 standards -- Keep models focused and single-purpose -- Use descriptive field names -- Use type hints for all fields -- Document complex validations -- Use Optional[] for nullable fields -- Use Field(default_factory=...) for mutable defaults diff --git a/.cursor/rules/check-pipes.mdc b/.cursor/rules/check-pipes.mdc deleted file mode 100644 index e34e137..0000000 --- a/.cursor/rules/check-pipes.mdc +++ /dev/null @@ -1,13 +0,0 @@ ---- -description: -globs: pipelex_libraries/**/*.plx -alwaysApply: false ---- -This rule automates checking the pipes all lint and load properly. - -When you modify the pipeline .plx files and/or the structures in the *.py files of the pipelex_libraries, you must check that they load properly. -Do the following: -1- Call `cocode validate` -2- Fetch logs from the console -3- Fix linting or validation issues - diff --git a/.cursor/rules/coding_standards.mdc b/.cursor/rules/coding_standards.mdc new file mode 100644 index 0000000..f8e7b29 --- /dev/null +++ b/.cursor/rules/coding_standards.mdc @@ -0,0 +1,124 @@ +--- +description: +globs: +alwaysApply: true +--- +# Coding Standards & Best Practices + +This document outlines the core coding standards, best practices, and quality control procedures for the codebase. + +## Type Hints + +1. **Always Use Type Hints** + - Every function parameter must be typed + - Every function return must be typed + - Use type hints for all variables where type is not obvious + - Use types with Uppercase first letter (Dict[], List[], etc.) + +2. **StrEnum** + - Import from `pipelex.types`: + ```python + from pipelex.types import StrEnum + ``` + +## BaseModel Standards + +- Respect Pydantic v2 standards +- Keep models focused and single-purpose +- Use descriptive field names +- Use type hints for all fields +- Document complex validations +- Use Optional[] for nullable fields +- Use Field(default_factory=...) for mutable defaults + +## Factory Pattern + +- Use Factory Pattern for object creation when dealing with multiple implementations + +## Documentation + +1. **Docstring Format** + ```python + def process_image(image_path: str, size: Tuple[int, int]) -> bytes: + """Process and resize an image. + + Args: + image_path: Path to the source image + size: Tuple of (width, height) for resizing + + Returns: + Processed image as bytes + """ + pass + ``` + +2. **Class Documentation** + ```python + class ImageProcessor: + """Handles image processing operations. + + Provides methods for resizing, converting, and optimizing images. + """ + ``` + +## Error Handling + +1. **Graceful Error Handling** + - Use try/except blocks with specific exceptions + - Convert third-party exceptions to custom ones + ```python + try: + from fal_client import AsyncClient as FalAsyncClient + except ImportError as exc: + raise MissingDependencyError( + "fal-client", "fal", + "The fal-client SDK is required to use FAL models." + ) from exc + ``` + +## Code Quality Checks + +### Linting and Type Checking + +Before finalizing a task, run: +```bash +make fix-unused-imports +make check +``` + +This runs multiple code quality tools: +- Pyright: Static type checking +- Ruff: Fast Python linter +- Mypy: Static type checker + +Always fix any issues reported by these tools before proceeding. + +### Running Tests + +1. **Quick Test Run** (no LLM/image generation): + ```bash + make tp + ``` + Runs tests with markers: `(dry_runnable or not (inference or llm or imgg or ocr)) and not (needs_output or pipelex_api)` + +2. **Specific Tests**: + ```bash + make tp TEST=TestClassName + # or + make tp TEST=test_function_name + ``` + Note: Matches names starting with the provided string. + +**Important**: Never run `make ti`, `make test-inference`, `make to`, `make test-ocr`, `make tg`, or `make test-imgg` - these use costly inference. + +## Pipelines + +- All pipeline definitions go in `cocode/pipelex_libraries/pipelines/` +- Always validate pipelines after creation/edit with `make validate`. + Iterate if there are errors. + +## Project Structure + +- **Pipelines**: `cocode/pipelex_libraries/pipelines/` +- **Tests**: `tests/` directory +- **Documentation**: `docs/` directory \ No newline at end of file diff --git a/.cursor/rules/docs.mdc b/.cursor/rules/docs.mdc new file mode 100644 index 0000000..31461aa --- /dev/null +++ b/.cursor/rules/docs.mdc @@ -0,0 +1,8 @@ +--- +description: +globs: docs/**/*.md +alwaysApply: false +--- +Write docs and answer questions about writing docs. + +We use Material for MkDocs. All markdown in our docs must be compatible with Material for MkDocs and done using best practices to get the best results with Material for MkDocs. diff --git a/.cursor/rules/llms.mdc b/.cursor/rules/llms.mdc index 2246c49..5b6d77c 100644 --- a/.cursor/rules/llms.mdc +++ b/.cursor/rules/llms.mdc @@ -1,12 +1,13 @@ --- -description: Use LLM models with approrpiate settings. Define LLM handles. Define LLM parameters directly in PipeLLM or through presets. -globs: +globs: *.plx,*.toml alwaysApply: false --- # Rules to choose LLM models used in PipeLLMs. +## LLM Handles + In order to use it in a pipe, an LLM is referenced by its llm_handle and possibly by an llm_preset. -Both llm_handles and llm_presets are defined in toml files in `pipelex_libraries/llm_deck`. The [base_llm_deck.toml](mdc:pipelex/libraries/llm_deck/base_llm_deck.toml) holds the standard presets, you must not touch this file. But you can add a new toml like `pipelex_libraries/llm_deck/some_domain.toml` and define new llm_presets there. +Both llm_handles and llm_presets are defined in this toml config file: [base_llm_deck.toml](./cocode/pipelex_libraries/llm_deck/base_llm_deck.toml) ## LLM Handles @@ -15,17 +16,21 @@ An llm_handle matches the handle (an id of sorts) with the full specification of - llm_version - llm_platform_choice -The declaration looks like this in toml syntax: +The declaration of llm_handles looks like this in toml syntax: ```toml [llm_handles] -gpt-4o-2024-08-06 = { llm_name = "gpt-4o", llm_version = "2024-08-06", llm_platform_choice = "openai" } +gpt-4o-2024-11-20 = { llm_name = "gpt-4o", llm_version = "2024-11-20" } ``` In mosty cases, we only want to use version "latest" and llm_platform_choice "default" in which case the declaration is simply a match of the llm_handle to the llm_name, like this: ```toml -best-claude = "claude-3-7-sonnet" +best-claude = "claude-4-opus" +best-gemini = "gemini-2.5-pro" +best-mistral = "mistral-large" ``` +And of course, llm_handles are automatically assigned for all models by their name, with version "latest" and llm_platform_choice "default". + ## Using an LLM Handle in a PipeLLM Here is an example of using an llm_handle to specify which LLM to use in a PipeLLM: @@ -36,12 +41,12 @@ type = "PipeLLM" definition = "Write text about Hello World." output = "Text" llm = { llm_handle = "gpt-4o-mini", temperature = 0.9, max_tokens = "auto" } -prompt = """ +prompt_template = """ Write a haiku about Hello World. """ ``` -As you can see, to use the LLM, you must also indicate the temperature (float between 0 and 1). But the max_tokens (either an int or the string "auto") is optional: by default, it uses "auto", i.e. no value passed, no limit applied. +As you can see, to use the LLM, you must also indicate the temperature (float between 0 and 1) and max_tokens (either an int or the string "auto"). ## LLM Presets @@ -49,8 +54,8 @@ Presets are meant to record the choice of an llm with its hyper parameters (temp Examples: ```toml -llm_to_reason = { llm_handle = "o4-mini", temperature = 1 } -llm_to_reason_on_diagram = { llm_handle = "best-claude", temperature = 0.5 } +llm_to_reason = { llm_handle = "o4-mini", temperature = 1, max_tokens = "auto" } +llm_to_extract_invoice = { llm_handle = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" } ``` The interest is that these presets can be used to set the LLM choice in a PipeLLM, like this: @@ -59,7 +64,7 @@ The interest is that these presets can be used to set the LLM choice in a PipeLL [pipe.extract_invoice] type = "PipeLLM" definition = "Extract invoice information from an invoice text transcript" -input = "InvoiceText" +inputs = { invoice_text = "InvoiceText" } output = "Invoice" llm = "llm_to_extract_invoice" prompt_template = """ @@ -73,3 +78,6 @@ The category of this invoice is: $invoice_details.category. The setting here `llm = "llm_to_extract_invoice"` works because "llm_to_extract_invoice" has been declared as an llm_preset in the deck. You must not use an LLM preset in a PipeLLM that does not exist in the deck. If needed, you can add llm presets. + + +You can override the predefined llm presets in [overrides.toml](../../cocode/pipelex_libraries/llm_deck/overrides.toml). diff --git a/.cursor/rules/pipelex.mdc b/.cursor/rules/pipelex.mdc index 3e1c9a8..ab21a1f 100644 --- a/.cursor/rules/pipelex.mdc +++ b/.cursor/rules/pipelex.mdc @@ -4,7 +4,7 @@ alwaysApply: true # Pipeline Guide - Always first write your "plan" in natural langage, then transcribe it in pipelex. -- You should ALWAYS RUN the terminal command `pipelex validate` when you are writing a `.plx` file. It will ensure the pipe is runnable. If not, iterate. +- You should ALWAYS RUN the terminal command `make validate` when you are writing a `.plx` file. It will ensure the pipe is runnable. If not, iterate. - Please use POSIX standard for files. (enmpty lines, no trailing whitespaces, etc.) # Pipeline Structure Guide @@ -96,6 +96,28 @@ inputs = { Concepts and their structure classes are meant to indicate an idea. A Concept MUST NEVER be a plural noun and you should never create a SomeConceptList: lists and arrays are implicitly handled by Pipelex according to the context. Just define SomeConcept. +**IMPORTANT: Never create unnecessary structure classes that only refine native concepts without adding fields.** + +DO NOT create structures like: +```python +class Joke(TextContent): + """A humorous text that makes people laugh.""" + pass +``` + +If a concept only refines a native concept (like Text, Image, etc.) without adding new fields, simply declare it in the .plx file: +```plx +[concept] +Joke = "A humorous text that makes people laugh." +``` +If you simply need to refine another native concept, construct it like this: +```plx +[concept.Landscape] +refines = "Image" +``` + +Only create a Python structure class when you need to add specific fields: + ```python from datetime import datetime from typing import List, Optional @@ -415,7 +437,7 @@ from pipelex.hub import get_pipeline_tracker, get_report_delegate from pipelex.pipelex import Pipelex from pipelex.pipeline.execute import execute_pipeline -from pipelex_libraries.pipelines.examples.extract_gantt.gantt import GanttChart +from cocode.pipelex_libraries.pipelines.examples.extract_gantt.gantt import GanttChart SAMPLE_NAME = "extract_gantt" IMAGE_URL = "assets/gantt/gantt_tree_house.png" @@ -515,6 +537,6 @@ So here are a few concrete examples of calls to execute_pipeline with various wa ) ``` -ALWAYS RUN `pipelex validate` when you are finished writing pipelines: This checks for errors. If there are errors, iterate until it works. +ALWAYS RUN `make validate` when you are finished writing pipelines: This checks for errors. If there are errors, iterate until it works. Then, create an example file to run the pipeline in the `examples` folder. But don't write documentation unless asked explicitly to. diff --git a/.cursor/rules/pipes.mdc b/.cursor/rules/pipes.mdc deleted file mode 100644 index 871c428..0000000 --- a/.cursor/rules/pipes.mdc +++ /dev/null @@ -1,224 +0,0 @@ ---- -description: -globs: **/pipelex_libraries/pipelines/**/*.plx -alwaysApply: false ---- -This rule explains how to build pipes. - -# File Naming & Structure - -## The pipelines/ directory - -- Pipelines and structures are defined in the `pipelex/libraries/pipelines` directory - -pipelex/libraries -└── pipelines - -## Pipeline file naming - -- Pipeline PLX files should be placed in the `pipelex/libraries/pipelines/` directory or a subdirectory of it -- The file name should be descriptive, in snake_case and end with `.plx` -- The file should contain both concepts and pipes definitions - -## Pipeline file structure - -1- Domain statement (name, definition, other attributes) -2- Concept definitions -3- Pipe definitions - -# Domain statement - -- Format: `domain = "domain_name"` -- Add a definition: `definition = "Description of the domain"` -- Note: the domain name is usually the plx filename for domains that fit in a single plx file. Otherwise, the domain name should be the name of the subdirectory where the files of the domain are gathered - -# Concept definitions - -- Start with `[concept]` header -- Naming: concepts are PascalCase, like python classes (e.g., `GanttTaskDetails`, `HtmlTable`) -- Define the concept, i.e. just state what it means. - - Never include the usage context in the concept. The concept indicates what the stuff is in itself, it's not "for something", it just is something. e.g. don't define "TextToSummarize": it's just "Text". If you want to refine the concept for instance you can define "Essay". - - In particular, never define a concept as a plural form: if the context of the pipeline execution makes it multiple, it will just be handled using a ListContent (see below). e.g. don't define a concept for "Stories", just define "Story". - - Also, avoid including adjectives in concepts, e.g. don't define "LargeText", it's just "Text". -- Don't redefine the native concepts from @concept_native.py - -⚠️ Important ⚠️ - -A Concept MUST NEVER be a plural noun and you should never create a SomeConceptList: lists and arrays are implicitly handled by Pipelex according to the context. Just define SomeConcept. - -- Define concepts in one of two ways: - -## Simple text-based concept definition - -```plx -[concept] -ConceptName = "Description of the concept" -``` - -## Detailed concept definition with `structure` and `refines` - -```plx -[concept.ConceptName] -Concept = "Description of the concept" -structure = "StructureName" -refines = "ParentConcept" # Optional, for concept inheritance -``` - -About the `structure` field: -- It's Optional -- It's the name of the Python BaseModel class used for the concept -- The class must be a subclass of StuffContent -- The class must be defined in a python module placed inside the `pipelex/libraries/pipelines/` directory -- If the `structure` field is omitted but a class with the same name as the concept is defined in the structures directory, then it's implicitly applied to the concept - -About the `refines` field: -- It's an Optional[List[str]] -- It indicates that the concept refines one or several other concepts: "refines" in the sense that it makes things more specific. e.g. "Dog" refines "Animal" and "Pet" -- As of this version, concept "refines" declaration are purely semantic guides - - -## Native structures - -Pipelex provides the following structures natively: - -- TextContent (that one is used if you don't specify any structure and the Concept's name does not exist as a structure class) -- NumberContent -- ImageContent -- HtmlContent -- MermaidContent -- LLMPromptContent - -The native structures are implied when using the native concepts: "Text", "Number", "Image", "PDF",... do you don't have the state it. - -Some subclasses of StuffContent exist which you should never use directly: -- ListContent: this is used internally to manipulate a list of stuff, but you should NEVER define a plural concept -- StructuredContent -- DynamicContent - -# Pipes Section - -- Start with `[pipe]` header -- Naming: pipe names are snake_case, like python functions (e.g., `extract_gantt_tasks`) -- Define pipes using as PipeLLM, PipeSequence, PipeParallel, PipeCondition - -# Full pipeline plx template - -Get inspiration from this template filled with appropriate examples: - -```plx -domain = "template" -definition = "Template file demonstrating different pipe types in Pipelex" - -[concept] -InputText = "A text input for processing" -OutputText = "A processed text output" -TableScreenshot = "An image of a table" -HtmlTable = "An HTML table representation" -Question = "A question to be answered" -FormattedAnswer = "A formatted answer to a question" -ExpenseReportText = "Text content of an expense report" -Composite = "A composite output containing multiple pieces of information" -Expense = "An expense record" -RulesToApply = "Rules to be applied to an expense" - -[pipe] ############## - -# Example of a PipeLLM that uses no input -[pipe.write_poem] -type = "PipeLLM" -definition = "Write a poem" -output = "Text" -llm = "llm_for_creative_writing" -prompt = """ -Write a poem about an AI that meets a Software and they fall in l0ve. -""" - -# Example of a PipeLLM that uses @ prefix to insert a block of text -[pipe.process_text] -type = "PipeLLM" -definition = "Process input text using an LLM" -inputs = { text = "Text" } -output = "Text" -llm = "llm_to_summarize_text" -system_prompt = """ -You are an expert in text summarization. -""" -prompt_template = """ -Summarize the following text: - -@text - -""" - -# Example of a PipeLLM that uses '@' prefix to insert a block of text but also a '$' prefix to insert text inline in a sentence (that is teh case for the $topic) -[pipe.summarize_topic] -type = "PipeLLM" -definition = "Summarize a dense text with of focus on a specific topic." -inputs = { topic = "Topic", text = "Text" } -output = "Summary" -prompt_template = """ -Your goal is to summarize everything related to $topic in the provided text: - -@text - -Please provide only the summary, with no additional text or explanations. -Your summary should not be longer than 2 sentences. -""" - -# Example of a PipeLLM with image vision processing by a VLM -[pipe.get_html_table_from_image] -type = "PipeLLM" -definition = "Convert table screenshot to HTML" -inputs = { table_screenshot = "TableScreenshot" } -output = "HtmlTable" -system_prompt = """ -You are a vision-based table extractor. -""" -prompt = """ -You are given a picture (screenshot) of a table, taken from a PDF document. -Your goal is to extract the table from the image **in html**. -Make sure you do not forget any text. Make sure you do not invent any text. -Make sure your merge is consistent. Make sure you replicate the formatting -(borders, text formatting, colors, text alignment...) -""" -llm = "llm_to_extract_tables" - -# Example of a PipeSequence -[pipe.answer_question_with_instructions] -type = "PipeSequence" -definition = "Answer a question with instructions" -inputs = { question = "Question" } -output = "FormattedAnswer" -steps = [ - { pipe = "enrich_instructions", result = "instructions", }, - { pipe = "answer_enriched_question", result = "answer" }, - { pipe = "format_answer", result = "formatted_answer" }, -] - -# Example of a PipeParallel -[pipe.extract_expense_report] -type = "PipeParallel" -definition = "Extract useful information from an expense report" -inputs = { expense_report = "ExpenseReportText" } -output = "Composite" -parallels = [ - { pipe = "extract_employee_from_expense_report", result = "employee" }, - { pipe = "extract_expenses_from_expense_report", result = "expenses" }, -] - -# Example of a PipeCondition -[pipe.expense_conditional_validation] -type = "PipeCondition" -definition = "Choose the rules to apply" -inputs = { expense = "Expense" } -output = "RulesToApply" -expression = "expense_category.category" -``` - -# Building pipes - -When building a pipe: -- First think about the logical steps you need to take. Then, plan the actual use of different patterns offered by Pipelex pipes, to apply them appropriately when you want to run things in sequence (PipeSequence) or in parallel, ot save time (PipeParallel). -- Think about what PipeLLM you are going to need and which LLM skills they will need. -- Write the pipe's plx and required structure in pipelex/pipelex_libraries/pipelines - diff --git a/.cursor/rules/piping.mdc b/.cursor/rules/piping.mdc deleted file mode 100644 index 90c26a1..0000000 --- a/.cursor/rules/piping.mdc +++ /dev/null @@ -1,21 +0,0 @@ ---- -description: -globs: -alwaysApply: false ---- -Rules to write pipes: - -- To write the pipes in the .plx files, check out: -[pipes.mdc](mdc:.cursor/rules/pipes.mdc) - -- To write the structure classes for concepts used in pipes with structured generations, check out: -[structures.mdc](mdc:.cursor/rules/structures.mdc) - -- To write prompts using our telmplating syntax, check out: -[prompt-templates.mdc](mdc:.cursor/rules/prompt-templates.mdc) - -- To choose an llm preset or define a new one, check out: -[llms.mdc](mdc:.cursor/rules/llms.mdc) - -- To check that all loads properly, check out: -[check-pipes.mdc](mdc:.cursor/rules/check-pipes.mdc) diff --git a/.cursor/rules/prompt-templates.mdc b/.cursor/rules/prompt-templates.mdc deleted file mode 100644 index 034f8c1..0000000 --- a/.cursor/rules/prompt-templates.mdc +++ /dev/null @@ -1,62 +0,0 @@ ---- -description: -globs: **/pipelex_libraries/pipelines/**/*.plx -alwaysApply: false ---- -This rule explains how to write prompt templates in PipeLLM definitions. These prompts will be rendered and become the user_text or the system_prompt part of LLM chat completion queries. - -# Jinja2 - -- We support jinja2 templates, so all the syntax capacities of jinja2 can be used such as variables inserted with double-curly-braces {{ like.this }} -- But we don't support jinja2 importing other templates: our templates only work as self-contained - -# Preprocessing shorthands - -Before handing over templates to jinja2, we preprocess them in order to support a simpler more readable syntax that will generate jinja2 syntax to insert variables and use appropriate filters. Our shorthand syntax is presented here: - -## Insert stuff inside a tagged block - -If the inserted text is supposedly long text, made of several lines or paragraphs, you want it inserted inside a block, possibly a block tagged and delimlited with proper syntax as one would do in a markdown documentation. To include stuff as a block, use the "@" prefix. - -Example template: -```plx -prompt_template = """ -Match the expense with its corresponding invoice: - -@expense - -@invoices -""" -``` -In this example, the expense data and the invoices data are obviously made of several lines each, that's why it makes sense to use the "@" prefix in order to have them delimited inside a block. Note that our preprocessor will automatically include the block's title, so it doens't need to be explictly written in the prompt template. - -**DO NOT write things like "Here is the expense: @expense".** -**DO write simply "@expense" alone in an isolated line.** - - -## Insert stuff inline - -If the inserted text is short text and it makes sense to have it inserted directly into a sentence, you want it inserted inline. To insert stuff inline, use the "$" prefix. This will insert the stuff without delimiters and the content will be rendered as plain text. - -Example template: -```plx -prompt_template = """ -Your goal is to summarize everything related to $topic in the provided text: - -@text - -Please provide only the summary, with no additional text or explanations. -Your summary should not be longer than 2 sentences. -""" -``` - -Here, $topic will be inserted inline, whereas @text will be a a delimited block. -Be sure to make the proper choice of prefix for each insertion. - -**DO NOT write "$topic" alone in an isolated line.** -**DO write things like "Write an essay about $topic" included in an actual sentence.** - -## Style rule - -Always favor the @ and $ prefixes instead of the jinja2 double-curly-brackets syntax, unless explicitly instructed otherwise. - diff --git a/.cursor/rules/pytest.mdc b/.cursor/rules/pytest.mdc index aab8c7c..3a0f72f 100644 --- a/.cursor/rules/pytest.mdc +++ b/.cursor/rules/pytest.mdc @@ -10,11 +10,15 @@ These rules apply when writing unit tests. - Name test files with `test_` prefix - Use descriptive names that match the functionality being tested -- Place test files in the appropriate subdirectory of `tests/`: - - `tests/cogt/` for tests related to sub-package `pipelex.cogt` - - `tests/tools/` for tests related to sub-package `pipelex.tools` - - `tests/pipelex/` for tests related to `pipelex`and not its sub-packages -- More precisely, for `pipelex` and `pipelex.cogt` place the tests inside subdirectories named either `asynch` for async test functions and `synchro` for normal non-async test functions +- Place test files in the appropriate test category directory: + - `tests/unit/` - for unit tests that test individual functions/classes in isolation + - `tests/integration/` - for integration tests that test component interactions + - `tests/e2e/` - for end-to-end tests that test complete workflows + - `tests/test_pipelines/` - for test pipeline definitions (PLX files and their structuring python files) +- Fixtures are defined in conftest.py modules at different levels of the hierarchy, their scope is handled by pytest +- Test data is placed inside test_data.py at different levels of the hierarchy, they must be imported with package paths from the root like `tests.pipelex.test_data`. Their content is all constants, regrouped inside classes to keep things tidy. +- Always put test inside Test classes. +- The pipelex pipelines should be stored in `tests/test_pipelines` as well as the related structured Output classes that inherit from `StructuredContent` ## Markers @@ -26,6 +30,10 @@ Apply the appropriate markers: Several markers may be applied. For instance, if the test uses an LLM, then it uses inference, so you must mark with both `inference`and `llm`. +## Tips + +- Never use the unittest.mock. Use pytest-mock + ## Test Class Structure Always group the tests of a module into a test class: @@ -68,7 +76,7 @@ from cocode.pipelex_libraries.pipelines.base_library.retrieve import RetrievedEx from pipelex.config_pipelex import get_config from pipelex.core.pipe import PipeAbstract, update_job_metadata_for_pipe -from pipelex.core.pipe_output import PipeOutput, PipeOutputType +from pipelex.core.pipes.pipe_output import PipeOutput, PipeOutputType from pipelex.core.pipes.pipe_run_params import PipeRunParams from pipelex.core.pipes.pipe_run_params import PipeRunParams from pipelex.pipe_works.pipe_router_protocol import PipeRouterProtocol @@ -98,12 +106,9 @@ working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff) ```python pipe_output: PipeOutput = await pipe_router.run_pipe( pipe_code="pipe_name", - pipe_run_params=PipeRunParams(), + pipe_run_params=PipeRunParamsFactory.make_run_params(), working_memory=working_memory, - job_metadata=JobMetadata( - session_id=get_config().session_id, - top_job_id=cast(str, request.node.originalname), # type: ignore - ), + job_metadata=JobMetadata(), ) ``` diff --git a/.cursor/rules/structures.mdc b/.cursor/rules/structures.mdc deleted file mode 100644 index ae87d32..0000000 --- a/.cursor/rules/structures.mdc +++ /dev/null @@ -1,63 +0,0 @@ ---- -description: -globs: -alwaysApply: false ---- -Rules to write structure classes for concepts used in pipes. -In particular, these structures are generated by PipeLLM using structured generation. - -# Structured Models Rules - -## Model Location and Registration - -- Create models for structured generations related to "some_domain" in `pipelex_libraries/pipelines/**/.py` -- Models must inherit from `StructuredContent` or another appropriate content type - -## Model Structure - -Concepts and their structure classes are meant to indicate an idea. -A Concept MUST NEVER be a plural noun and you should never create a SomeConceptList: lists and arrays are implicitly handled by Pipelex according to the context. Just define SomeConcept. - -```python -from datetime import datetime -from typing import List, Optional -from pydantic import BaseModel, Field - -from pipelex.core.stuffs.stuff_content import StructuredContent - -class YourModel(StructuredContent): - # Required fields - field1: str - field2: int - - # Optional fields with defaults - field3: Optional[str] = Field(None, "Description of field3") - field4: List[str] = Field(default_factory=list) - - # Date fields should remove timezone - date_field: Optional[datetime] = None - - @field_validator("date_field") - @classmethod - def remove_tzinfo(cls, v: Optional[datetime]) -> Optional[datetime]: - if v is not None: - return v.replace(tzinfo=None) - return v -``` - -## Usage - -Structure classes are meant to indicate what class to use for a particular Concept. In general they use the same name as the concept. - -Structure classes defined within `pipelex_libraries/pipelines/` are automatically loaded into the class_registry when setting up Pipelex, not need to do it manually. - -## Best Practices for structures - -- Respect Pydantic v2 standards -- Use type hints for all fields -- Use `Field` declaration and write the description -- Use Pydantic validators for data cleaning/validation -- Remove timezone info from datetime fields -- Respect rules of [pydantic.mdc](mdc:.cursor/rules/pydantic.mdc) - - diff --git a/.cursor/rules/tdd.mdc b/.cursor/rules/tdd.mdc new file mode 100644 index 0000000..575544a --- /dev/null +++ b/.cursor/rules/tdd.mdc @@ -0,0 +1,28 @@ +--- +description: +globs: +alwaysApply: false +--- +# Test-Driven Development Guide + +This document outlines our test-driven development (TDD) process and the tools available for testing. + +## TDD Cycle + +1. **Write a Test First** +[pytest.mdc](pytest.mdc) + +2. **Write the Code** + - Implement the minimum amount of code needed to pass the test + - Follow the project's coding standards + - Keep it simple - don't write more than needed + +3. **Run Linting and Type Checking** +[coding_standards.mdc](coding_standards.mdc) + +4. **Refactor if needed** +If the code needs refactoring, with the best practices [coding_standards.mdc](coding_standards.mdc) + +5. **Validate tests** + +Remember: The key to TDD is writing the test first and letting it drive your implementation. Always run the full test suite and quality checks before considering a feature complete. diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..993a025 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,926 @@ +Concatenation +# Coding Standards & Best Practices + +This document outlines the core coding standards, best practices, and quality control procedures for the codebase. + +## Type Hints + +1. **Always Use Type Hints** + - Every function parameter must be typed + - Every function return must be typed + - Use type hints for all variables where type is not obvious + - Use types with Uppercase first letter (Dict[], List[], etc.) + +2. **StrEnum** + - Import from `pipelex.types`: + ```python + from pipelex.types import StrEnum + ``` + +## BaseModel Standards + +- Respect Pydantic v2 standards +- Keep models focused and single-purpose +- Use descriptive field names +- Use type hints for all fields +- Document complex validations +- Use Optional[] for nullable fields +- Use Field(default_factory=...) for mutable defaults + +## Factory Pattern + +- Use Factory Pattern for object creation when dealing with multiple implementations + +## Documentation + +1. **Docstring Format** + ```python + def process_image(image_path: str, size: Tuple[int, int]) -> bytes: + """Process and resize an image. + + Args: + image_path: Path to the source image + size: Tuple of (width, height) for resizing + + Returns: + Processed image as bytes + """ + pass + ``` + +2. **Class Documentation** + ```python + class ImageProcessor: + """Handles image processing operations. + + Provides methods for resizing, converting, and optimizing images. + """ + ``` + +## Error Handling + +1. **Graceful Error Handling** + - Use try/except blocks with specific exceptions + - Convert third-party exceptions to custom ones + ```python + try: + from fal_client import AsyncClient as FalAsyncClient + except ImportError as exc: + raise MissingDependencyError( + "fal-client", "fal", + "The fal-client SDK is required to use FAL models." + ) from exc + ``` + +## Code Quality Checks + +### Linting and Type Checking + +Before finalizing a task, run: +```bash +make fix-unused-imports +make check +``` + +This runs multiple code quality tools: +- Pyright: Static type checking +- Ruff: Fast Python linter +- Mypy: Static type checker + +Always fix any issues reported by these tools before proceeding. + +### Running Tests + +1. **Quick Test Run** (no LLM/image generation): + ```bash + make tp + ``` + Runs tests with markers: `(dry_runnable or not (inference or llm or imgg or ocr)) and not (needs_output or pipelex_api)` + +2. **Specific Tests**: + ```bash + make tp TEST=TestClassName + # or + make tp TEST=test_function_name + ``` + Note: Matches names starting with the provided string. + +**Important**: Never run `make ti`, `make test-inference`, `make to`, `make test-ocr`, `make tg`, or `make test-imgg` - these use costly inference. + +## Pipelines + +- All pipeline definitions go in `cocode/pipelex_libraries/pipelines/` +- Always validate pipelines after creation/edit with `make validate`. + Iterate if there are errors. + +## Project Structure + +- **Pipelines**: `cocode/pipelex_libraries/pipelines/` +- **Tests**: `tests/` directory +- **Documentation**: `docs/` directory +# Pipeline Guide + +- Always first write your "plan" in natural langage, then transcribe it in pipelex. +- You should ALWAYS RUN the terminal command `make validate` when you are writing a `.plx` file. It will ensure the pipe is runnable. If not, iterate. +- Please use POSIX standard for files. (enmpty lines, no trailing whitespaces, etc.) + +# Pipeline Structure Guide + +## Pipeline File Naming +- Files must be `.plx` for pipelines (Always add an empty line at the end of the file, and do not add trailing whitespaces to PLX files at all) +- Files must be `.py` for structures +- Use descriptive names in `snake_case` + +## Pipeline File Structure +A pipeline file has three main sections: +1. Domain statement +2. Concept definitions +3. Pipe definitions + +### Domain Statement +```plx +domain = "domain_name" +definition = "Description of the domain" # Optional +``` +Note: The domain name usually matches the plx filename for single-file domains. For multi-file domains, use the subdirectory name. + +### Concept Definitions +```plx +[concept] +ConceptName = "Description of the concept" # Should be the same name as the Structure ClassName you want to output +``` + +Important Rules: +- Use PascalCase for concept names +- Never use plurals (no "Stories", use "Story") +- Avoid adjectives (no "LargeText", use "Text") +- Don't redefine native concepts (Text, Image, PDF, TextAndImages, Number) +yes +### Pipe Definitions + +## Pipe Base Structure + +```plx +[pipe.your_pipe_name] +type = "PipeLLM" +definition = "A description of what your pipe does" +inputs = { input_1 = "ConceptName1", input_2 = "ConceptName2" } +output = "ConceptName" +``` + +DO NOT WRITE: +```plx +[pipe.your_pipe_name] +type = "pipe_sequence" +``` + +But it should be: + +```plx +[pipe.your_pipe_name] +type = "PipeSequence" +definition = "....." +``` + +The pipes will all have at least this base structure. +- `inputs`: Dictionnary of key behing the variable used in the prompts, and the value behing the ConceptName. It should ALSO LIST THE INPUTS OF THE INTERMEDIATE STEPS (if pipeSequence) or of the conditionnal pipes (if pipeCondition). +So If you have this error: +`StaticValidationError: missing_input_variable • domain='expense_validator' • pipe='validate_expense' • +variable='['ocr_input']'`` +That means that the pipe validate_expense is missing the input `ocr_input` because one of the subpipe is needing it. + +NEVER WRITE THE INPUTS BY BREAKING THE LINE LIKE THIS: + +```plx +inputs = { + input_1 = "ConceptName1", + input_2 = "ConceptName2" +} +``` + + +- `output`: The name of the concept to output. The `ConceptName` should have the same name as the python class if you want structured output: + +# Structured Models Rules + +## Model Location and Registration + +- Create models for structured generations related to "some_domain" in `pipelex_libraries/pipelines/.py` +- Models must inherit from `StructuredContent` or appropriate content type + +## Model Structure + +Concepts and their structure classes are meant to indicate an idea. +A Concept MUST NEVER be a plural noun and you should never create a SomeConceptList: lists and arrays are implicitly handled by Pipelex according to the context. Just define SomeConcept. + +**IMPORTANT: Never create unnecessary structure classes that only refine native concepts without adding fields.** + +DO NOT create structures like: +```python +class Joke(TextContent): + """A humorous text that makes people laugh.""" + pass +``` + +If a concept only refines a native concept (like Text, Image, etc.) without adding new fields, simply declare it in the .plx file: +```plx +[concept] +Joke = "A humorous text that makes people laugh." +``` +If you simply need to refine another native concept, construct it like this: +```plx +[concept.Landscape] +refines = "Image" +``` + +Only create a Python structure class when you need to add specific fields: + +```python +from datetime import datetime +from typing import List, Optional +from pydantic import Field + +from pipelex.core.stuffs.stuff_content import StructuredContent + +# IMPORTANT: THE CLASS MUST BE A SUBCLASS OF StructuredContent +class YourModel(StructuredContent): # Always be a subclass of StructuredContent + # Required fields + field1: str + field2: int + + # Optional fields with defaults + field3: Optional[str] = Field(None, "Description of field3") + field4: List[str] = Field(default_factory=list) + + # Date fields should remove timezone + date_field: Optional[datetime] = None +``` +## Usage + +Structures are meant to indicate what class to use for a particular Concept. In general they use the same name as the concept. + +Structure classes defined within `pipelex_libraries/pipelines/` are automatically loaded into the class_registry when setting up Pipelex, no need to do it manually. + + +## Best Practices for structures + +- Respect Pydantic v2 standards +- Use type hints for all fields +- Use `Field` declaration and write the description + + +## Pipe Controllers and Pipe Operator + +Look at the Pipes we have in order to adapt it. Pipes are organized in two categories: + +1. **Controllers** - For flow control: + - `PipeSequence` - For creating a sequence of multiple steps + - `PipeCondition` - If the next pipe depends of the expression of a stuff in the working memory + - `PipeParallel` - For parallelizing pipes + - `PipeBatch` - For running pipes in Batch over a ListContent + +2. **Operators** - For specific tasks: + - `PipeLLM` - Generate Text and Objects (include Vision LLM) + - `PipeOcr` - OCR Pipe + - `PipeImgGen` - Generate Images + - `PipeFunc` - For running classic python scripts + +# PipeSequence Guide + +## Purpose +PipeSequence executes multiple pipes in a defined order, where each step can use results from previous steps. + +## Basic Structure +```plx +[pipe.your_sequence_name] +type = "PipeSequence" +definition = "Description of what this sequence does" +inputs = { input_name = "InputType" } # All the inputs of the sub pipes, except the ones generated by intermediate steps +output = "OutputType" +steps = [ + { pipe = "first_pipe", result = "first_result" }, + { pipe = "second_pipe", result = "second_result" }, + { pipe = "final_pipe", result = "final_result" } +] +``` + +## Key Components + +1. **Steps Array**: List of pipes to execute in sequence + - `pipe`: Name of the pipe to execute + - `result`: Name to assign to the pipe's output that will be in the working memory + +## Using PipeBatch in Steps + +You can use PipeBatch functionality within steps using `batch_over` and `batch_as`: + +```plx +steps = [ + { pipe = "process_items", batch_over = "input_list", batch_as = "current_item", result = "processed_items" + } +] +``` + +1. **batch_over**: Specifies a `ListContent` field to iterate over. Each item in the list will be processed individually and IN PARALLEL by the pipe. + - Must be a `ListContent` type containing the items to process + - Can reference inputs or results from previous steps + +2. **batch_as**: Defines the name that will be used to reference the current item being processed + - This name can be used in the pipe's input mappings + - Makes each item from the batch available as a single element + +The result of a batched step will be a `ListContent` containing the outputs from processing each item. + +# PipeCondition Controller + +The PipeCondition controller allows you to implement conditional logic in your pipeline, choosing which pipe to execute based on an evaluated expression. It supports both direct expressions and expression templates. + +## Usage in PLX Configuration + +### Basic Usage with Direct Expression + +```plx +[pipe.conditional_operation] +type = "PipeCondition" +definition = "A conditonal pipe to decide wheter..." +inputs = { input_data = "CategoryInput" } +output = "native.Text" +expression = "input_data.category" + +[pipe.conditional_operation.pipe_map] +small = "process_small" +medium = "process_medium" +large = "process_large" +``` +or +```plx +[pipe.conditional_operation] +type = "PipeCondition" +definition = "A conditonal pipe to decide wheter..." +inputs = { input_data = "CategoryInput" } +output = "native.Text" +expression_template = "{{ input_data.category }}" # Jinja2 code + +[pipe.conditional_operation.pipe_map] +small = "process_small" +medium = "process_medium" +large = "process_large" +``` + +## Key Parameters + +- `expression`: Direct boolean or string expression (mutually exclusive with expression_template) +- `expression_template`: Jinja2 template for more complex conditional logic (mutually exclusive with expression) +- `pipe_map`: Dictionary mapping expression results to pipe codes : +1 - The key on the left (`small`, `medium`) is the result of `expression` or `expression_template`. +2 - The value on the right (`process_small`, `process_medium`, ..) is the name of the pipce to trigger + +# PipeBatch Controller + +The PipeBatch controller allows you to apply a pipe operation to each element in a list of inputs in parallele. It is created via a PipeSequence. + +## Usage in PLX Configuration + +```plx +[pipe.sequence_with_batch] +type = "PipeSequence" +definition = "A Sequence of pipes" +inputs = { input_data = "ConceptName" } +output = "OutputConceptName" +steps = [ + { pipe = "pipe_to_apply", batch_over = "input_list", batch_as = "current_item", result = "batch_results" } +] +``` + +## Key Parameters + +- `pipe`: The pipe operation to apply to each element in the batch +- `batch_over`: The name of the list in the context to iterate over +- `batch_as`: The name to use for the current element in the pipe's context +- `result`: Where to store the results of the batch operation + +# PipeLLM Guide + +## Purpose + +PipeLLM is used to: +1. Generate text or objects with LLMs +2. Process images with Vision LLMs + +## Basic Usage + +### Simple Text Generation +```plx +[pipe.write_story] +type = "PipeLLM" +definition = "Write a short story" +output = "Text" +prompt_template = """ +Write a short story about a programmer. +""" +``` + +### Structured Data Extraction +```plx +[pipe.extract_info] +type = "PipeLLM" +definition = "Extract information" +inputs = { text = "Text" } +output = "PersonInfo" +prompt_template = """ +Extract person information from this text: +@text +""" +``` + +### System Prompts +Add system-level instructions: +```plx +[pipe.expert_analysis] +type = "PipeLLM" +definition = "Expert analysis" +output = "Analysis" +system_prompt = "You are a data analysis expert" +prompt_template = "Analyze this data" +``` + +### Multiple Outputs +Generate multiple results: +```plx +[pipe.generate_ideas] +type = "PipeLLM" +definition = "Generate ideas" +output = "Idea" +nb_output = 3 # Generate exactly 3 ideas +# OR +multiple_output = true # Let the LLM decide how many to generate +``` + +### Vision Tasks +Process images with VLMs: +```plx +[pipe.analyze_image] +type = "PipeLLM" +definition = "Analyze image" +inputs = { image = "Image" } # `image` is the name of the stuff that contains the Image. If its in a stuff, you can add something like `{ "page.image": "Image" } +output = "ImageAnalysis" +prompt_template = "Describe what you see in this image" +``` + +# PipeOCR Guide + +## Purpose + +Extract text and images from an image or a PDF + +## Basic Usage + +### Simple Text Generation +```plx +[pipe.extract_info] +type = "PipeOcr" +definition = "extract the information" +inputs = { ocr_input = "PDF" } # or { ocr_input = "Image" } if its an image. This is the only input +output = "Page" +``` + +The input ALWAYS HAS TO BE `ocr_input` and the value is either of concept `Image` or `Pdf`. + +The output concept `Page` is a native concept, with the structure `PageContent`: +It corresponds to 1 page. Therefore, the PipeOcr is outputing a `ListContent` of `Page` + +```python +class TextAndImagesContent(StuffContent): + text: Optional[TextContent] + images: Optional[List[ImageContent]] + +class PageContent(StructuredContent): # CONCEPT IS "Page" + text_and_images: TextAndImagesContent + page_view: Optional[ImageContent] = None +``` +- `text_and_images` are the text, and the related images found in the input image or PDF. +- `page_view` is the screenshot of the whole pdf page/image. + +This rule explains how to write prompt templates in PipeLLM definitions. + +## Insert stuff inside a tagged block + +If the inserted text is supposedly long text, made of several lines or paragraphs, you want it inserted inside a block, possibly a block tagged and delimlited with proper syntax as one would do in a markdown documentation. To include stuff as a block, use the "@" prefix. + +Example template: +```plx +prompt_template = """ +Match the expense with its corresponding invoice: + +@expense + +@invoices +""" +``` +In this example, the expense data and the invoices data are obviously made of several lines each, that's why it makes sense to use the "@" prefix in order to have them delimited inside a block. Note that our preprocessor will automatically include the block's title, so it doens't need to be explictly written in the prompt template. + +**DO NOT write things like "Here is the expense: @expense".** +**DO write simply "@expense" alone in an isolated line.** + +## Insert stuff inline + +If the inserted text is short text and it makes sense to have it inserted directly into a sentence, you want it inserted inline. To insert stuff inline, use the "$" prefix. This will insert the stuff without delimiters and the content will be rendered as plain text. + +Example template: +```plx +prompt_template = """ +Your goal is to summarize everything related to $topic in the provided text: + +@text + +Please provide only the summary, with no additional text or explanations. +Your summary should not be longer than 2 sentences. +""" +``` + +Here, $topic will be inserted inline, whereas @text will be a a delimited block. +Be sure to make the proper choice of prefix for each insertion. + +**DO NOT write "$topic" alone in an isolated line.** +**DO write things like "Write an essay about $topic" included in an actual sentence.** + +# Example to execute a pipeline + +```python +import asyncio + +from pipelex import pretty_print +from pipelex.hub import get_pipeline_tracker, get_report_delegate +from pipelex.pipelex import Pipelex +from pipelex.pipeline.execute import execute_pipeline + +from cocode.pipelex_libraries.pipelines.examples.extract_gantt.gantt import GanttChart + +SAMPLE_NAME = "extract_gantt" +IMAGE_URL = "assets/gantt/gantt_tree_house.png" + + +async def extract_gantt(image_url: str) -> GanttChart: + # Run the pipe + pipe_output = await execute_pipeline( + pipe_code="extract_gantt_by_steps", + input_memory={ + "gantt_chart_image": { + "concept": "gantt.GanttImage", + "content": ImageContent(url=image_url), + } + }, + ) + # Output the result + return pipe_output.main_stuff_as(content_type=GanttChart) + + +# start Pipelex +Pipelex.make() + +# run sample using asyncio +gantt_chart = asyncio.run(extract_gantt(IMAGE_URL)) + +# Display cost report (tokens used and cost) +get_report_delegate().generate_report() +# output results +pretty_print(gantt_chart, title="Gantt Chart") +get_pipeline_tracker().output_flowchart() +``` + +The input memory is a dictionary of key-value pairs, where the key is the name of the input variable and the value provides details to make it a stuff object. The relevant definitions are: +```python +StuffContentOrData = Dict[str, Any] | StuffContent | List[Any] | str +ImplicitMemory = Dict[str, StuffContentOrData] +``` +As you can seen, we made it so different ways can be used to define that stuff using structured content or data. + +So here are a few concrete examples of calls to execute_pipeline with various ways to set up the input memory: + +```python +# Here we have a single input and it's a Text. +# If you assign a string, by default it will be considered as a TextContent. + pipe_output = await execute_pipeline( + pipe_code="master_advisory_orchestrator", + input_memory={ + "user_input": problem_description, + }, + ) + +# Here we have a single input and it's a PDF. +# Because PDFContent is a native concept, we can use it directly as a value, +# the system knows what content it corresponds to: + pipe_output = await execute_pipeline( + pipe_code="power_extractor_dpe", + input_memory={ + "ocr_input": PDFContent(url=pdf_url), + }, + ) + +# Here we have a single input and it's an Image. +# Because ImageContent is a native concept, we can use it directly as a value: + pipe_output = await execute_pipeline( + pipe_code="fashion_variation_pipeline", + input_memory={ + "fashion_photo": ImageContent(url=image_url), + }, + ) + +# Here we have a single input, it's an image but +# its actually a more specific concept gantt.GanttImage which refines Image, +# so we must provide it using a dict with the concept and the content: + pipe_output = await execute_pipeline( + pipe_code="extract_gantt_by_steps", + input_memory={ + "gantt_chart_image": { + "concept": "gantt.GanttImage", + "content": ImageContent(url=image_url), + } + }, + ) + +# Here is a more complex example with multiple inputs assigned using different ways: + pipe_output = await execute_pipeline( + pipe_code="retrieve_then_answer", + dynamic_output_concept_code="contracts.Fees", + input_memory={ + "text": load_text_from_path(path=text_path), + "question": { + "concept": "answer.Question", + "content": question, + }, + "client_instructions": client_instructions, + }, + ) +``` + +ALWAYS RUN `make validate` when you are finished writing pipelines: This checks for errors. If there are errors, iterate until it works. +Then, create an example file to run the pipeline in the `examples` folder. +But don't write documentation unless asked explicitly to. + +# Rules to choose LLM models used in PipeLLMs. + +## LLM Handles + +In order to use it in a pipe, an LLM is referenced by its llm_handle and possibly by an llm_preset. +Both llm_handles and llm_presets are defined in this toml config file: [base_llm_deck.toml](./cocode/pipelex_libraries/llm_deck/base_llm_deck.toml) + +## LLM Handles + +An llm_handle matches the handle (an id of sorts) with the full specification of the LLM to use, i.e.: +- llm_name +- llm_version +- llm_platform_choice + +The declaration of llm_handles looks like this in toml syntax: +```toml +[llm_handles] +gpt-4o-2024-11-20 = { llm_name = "gpt-4o", llm_version = "2024-11-20" } +``` + +In mosty cases, we only want to use version "latest" and llm_platform_choice "default" in which case the declaration is simply a match of the llm_handle to the llm_name, like this: +```toml +best-claude = "claude-4-opus" +best-gemini = "gemini-2.5-pro" +best-mistral = "mistral-large" +``` + +And of course, llm_handles are automatically assigned for all models by their name, with version "latest" and llm_platform_choice "default". + +## Using an LLM Handle in a PipeLLM + +Here is an example of using an llm_handle to specify which LLM to use in a PipeLLM: + +```plx +[pipe.hello_world] +type = "PipeLLM" +definition = "Write text about Hello World." +output = "Text" +llm = { llm_handle = "gpt-4o-mini", temperature = 0.9, max_tokens = "auto" } +prompt_template = """ +Write a haiku about Hello World. +""" +``` + +As you can see, to use the LLM, you must also indicate the temperature (float between 0 and 1) and max_tokens (either an int or the string "auto"). + +## LLM Presets + +Presets are meant to record the choice of an llm with its hyper parameters (temperature and max_tokens) if it's good for a particular task. LLM Presets are skill-oriented. + +Examples: +```toml +llm_to_reason = { llm_handle = "o4-mini", temperature = 1, max_tokens = "auto" } +llm_to_extract_invoice = { llm_handle = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" } +``` + +The interest is that these presets can be used to set the LLM choice in a PipeLLM, like this: + +```plx +[pipe.extract_invoice] +type = "PipeLLM" +definition = "Extract invoice information from an invoice text transcript" +inputs = { invoice_text = "InvoiceText" } +output = "Invoice" +llm = "llm_to_extract_invoice" +prompt_template = """ +Extract invoice information from this invoice: + +The category of this invoice is: $invoice_details.category. + +@invoice_text +""" +``` + +The setting here `llm = "llm_to_extract_invoice"` works because "llm_to_extract_invoice" has been declared as an llm_preset in the deck. +You must not use an LLM preset in a PipeLLM that does not exist in the deck. If needed, you can add llm presets. + + +You can override the predefined llm presets in [overrides.toml](./cocode/pipelex_libraries/llm_deck/overrides.toml). + +These rules apply when writing unit tests. +- Always use pytest + +## Test file structure + +- Name test files with `test_` prefix +- Use descriptive names that match the functionality being tested +- Place test files in the appropriate test category directory: + - `tests/unit/` - for unit tests that test individual functions/classes in isolation + - `tests/integration/` - for integration tests that test component interactions + - `tests/e2e/` - for end-to-end tests that test complete workflows + - `tests/test_pipelines/` - for test pipeline definitions (PLX files and their structuring python files) +- Fixtures are defined in conftest.py modules at different levels of the hierarchy, their scope is handled by pytest +- Test data is placed inside test_data.py at different levels of the hierarchy, they must be imported with package paths from the root like `tests.pipelex.test_data`. Their content is all constants, regrouped inside classes to keep things tidy. +- Always put test inside Test classes. +- The pipelex pipelines should be stored in `tests/test_pipelines` as well as the related structured Output classes that inherit from `StructuredContent` + +## Markers + +Apply the appropriate markers: +- "llm: uses an LLM to generate text or objects" +- "imgg: uses an image generation AI" +- "inference: uses either an LLM or an image generation AI" +- "gha_disabled: will not be able to run properly on GitHub Actions" + +Several markers may be applied. For instance, if the test uses an LLM, then it uses inference, so you must mark with both `inference`and `llm`. + +## Tips + +- Never use the unittest.mock. Use pytest-mock + +## Test Class Structure + +Always group the tests of a module into a test class: + +```python +@pytest.mark.llm +@pytest.mark.inference +@pytest.mark.asyncio(loop_scope="class") +class TestFooBar: + @pytest.mark.parametrize( + "topic test_case_blueprint", + [ + TestCases.CASE_1, + TestCases.CASE_2, + ], + ) + async def test_pipe_processing( + self, + request: FixtureRequest, + topic: str, + test_case_blueprint: StuffBlueprint, + ): + # Test implementation +``` + +Sometimes it can be convenient to access the test's name in its body, for instance to include into a job_id. To achieve that, add the argument `request: FixtureRequest` into the signature and then you can get the test name using `cast(str, request.node.originalname), # type: ignore`. + +# Pipe tests + +## Required imports for pipe tests + +```python +import pytest +from pytest import FixtureRequest +from pipelex import log, pretty_print +from pipelex.core.stuffs.stuff_factory import StuffBlueprint, StuffFactory +from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory +from pipelex.hub import get_report_delegate +from cocode.pipelex_libraries.pipelines.base_library.retrieve import RetrievedExcerpt +from pipelex.config_pipelex import get_config + +from pipelex.core.pipe import PipeAbstract, update_job_metadata_for_pipe +from pipelex.core.pipes.pipe_output import PipeOutput, PipeOutputType +from pipelex.core.pipes.pipe_run_params import PipeRunParams +from pipelex.core.pipes.pipe_run_params import PipeRunParams +from pipelex.pipe_works.pipe_router_protocol import PipeRouterProtocol +``` + +## Pipe test implementation steps + +1. Create Stuff from blueprint: + +```python +stuff = StuffFactory.make_stuff( + concept_code="RetrievedExcerpt", + domain="retrieve", + content=RetrievedExcerpt(text="", justification="") + name="retrieved_text", +) +``` + +2. Create Working Memory: + +```python +working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff) +``` + +3. Run the pipe: + +```python +pipe_output: PipeOutput = await pipe_router.run_pipe( + pipe_code="pipe_name", + pipe_run_params=PipeRunParamsFactory.make_run_params(), + working_memory=working_memory, + job_metadata=JobMetadata(), +) +``` + +4. Log output and generate report: + +```python +pretty_print(pipe_output, title=f"Pipe output") +get_report_delegate().generate_report() +``` + +5. Basic assertions: + +```python +assert pipe_output is not None +assert pipe_output.working_memory is not None +assert pipe_output.main_stuff is not None +``` + +## Test Data Organization + +- If it's not already there, create a `test_data.py` file in the test directory +- Define test cases using `StuffBlueprint`: + +```python +class TestCases: + CASE_BLUEPRINT_1 = StuffBlueprint( + name="test_case_1", + concept_code="domain.ConceptName1", + value="test_value" + ) + CASE_BLUEPRINT_2 = StuffBlueprint( + name="test_case_2", + concept_code="domain.ConceptName2", + value="test_value" + ) + + CASE_BLUEPRINTS: ClassVar[List[Tuple[str, str]]] = [ # topic, blueprint" + ("topic1", CASE_BLUEPRINT_1), + ("topic2", CASE_BLUEPRINT_2), + ] +``` + +Note how we avoid initializing a default mutable value within a class instance, instead we use ClassVar. +Also note that we provide a topic for the test case, which is purely for convenience. + +## Best Practices for Testing + +- Use parametrize for multiple test cases +- Test both success and failure cases +- Verify working memory state +- Check output structure and content +- Use meaningful test case names +- Include docstrings explaining test purpose +- Log outputs for debugging +- Generate reports for cost tracking + +# Test-Driven Development Guide + +This document outlines our test-driven development (TDD) process and the tools available for testing. + +## TDD Cycle + +1. **Write a Test First** +[pytest.mdc](pytest.mdc) + +2. **Write the Code** + - Implement the minimum amount of code needed to pass the test + - Follow the project's coding standards + - Keep it simple - don't write more than needed + +3. **Run Linting and Type Checking** +[coding_standards.mdc](coding_standards.mdc) + +4. **Refactor if needed** +If the code needs refactoring, with the best practices [coding_standards.mdc](coding_standards.mdc) + +5. **Validate tests** + +Remember: The key to TDD is writing the test first and letting it drive your implementation. Always run the full test suite and quality checks before considering a feature complete. + diff --git a/AGENTS.md b/AGENTS.md index b72fef0..993a025 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,44 +1,780 @@ -# General rules +Concatenation +# Coding Standards & Best Practices -## Repo structure +This document outlines the core coding standards, best practices, and quality control procedures for the codebase. -The purpose of pipelex-template is to kick-start porjects based on the `pipelex` library for low-code AI workflows for repeatable processes. +## Type Hints -## Code Style & formatting +1. **Always Use Type Hints** + - Every function parameter must be typed + - Every function return must be typed + - Use type hints for all variables where type is not obvious + - Use types with Uppercase first letter (Dict[], List[], etc.) -- Imitate existing style -- Use type hints -- Respect Pydantic v2 standard -- Use Typer for CLIs -- Use explicit keyword arguments for function calls with multiple parameters (e.g., `func(arg_name=value)` not just `func(value)`) -- Add trailing commas to multi-line lists, dicts, function arguments, and tuples with >2 items (helps with cleaner diffs and prevents syntax errors when adding items) -- All imports inside this repo's packages must be absolute package paths from the root +2. **StrEnum** + - Import from `pipelex.types`: + ```python + from pipelex.types import StrEnum + ``` -## Writing tests +## BaseModel Standards +- Respect Pydantic v2 standards +- Keep models focused and single-purpose +- Use descriptive field names +- Use type hints for all fields +- Document complex validations +- Use Optional[] for nullable fields +- Use Field(default_factory=...) for mutable defaults + +## Factory Pattern + +- Use Factory Pattern for object creation when dealing with multiple implementations + +## Documentation + +1. **Docstring Format** + ```python + def process_image(image_path: str, size: Tuple[int, int]) -> bytes: + """Process and resize an image. + + Args: + image_path: Path to the source image + size: Tuple of (width, height) for resizing + + Returns: + Processed image as bytes + """ + pass + ``` + +2. **Class Documentation** + ```python + class ImageProcessor: + """Handles image processing operations. + + Provides methods for resizing, converting, and optimizing images. + """ + ``` + +## Error Handling + +1. **Graceful Error Handling** + - Use try/except blocks with specific exceptions + - Convert third-party exceptions to custom ones + ```python + try: + from fal_client import AsyncClient as FalAsyncClient + except ImportError as exc: + raise MissingDependencyError( + "fal-client", "fal", + "The fal-client SDK is required to use FAL models." + ) from exc + ``` + +## Code Quality Checks + +### Linting and Type Checking + +Before finalizing a task, run: +```bash +make fix-unused-imports +make check +``` + +This runs multiple code quality tools: +- Pyright: Static type checking +- Ruff: Fast Python linter +- Mypy: Static type checker + +Always fix any issues reported by these tools before proceeding. + +### Running Tests + +1. **Quick Test Run** (no LLM/image generation): + ```bash + make tp + ``` + Runs tests with markers: `(dry_runnable or not (inference or llm or imgg or ocr)) and not (needs_output or pipelex_api)` + +2. **Specific Tests**: + ```bash + make tp TEST=TestClassName + # or + make tp TEST=test_function_name + ``` + Note: Matches names starting with the provided string. + +**Important**: Never run `make ti`, `make test-inference`, `make to`, `make test-ocr`, `make tg`, or `make test-imgg` - these use costly inference. + +## Pipelines + +- All pipeline definitions go in `cocode/pipelex_libraries/pipelines/` +- Always validate pipelines after creation/edit with `make validate`. + Iterate if there are errors. + +## Project Structure + +- **Pipelines**: `cocode/pipelex_libraries/pipelines/` +- **Tests**: `tests/` directory +- **Documentation**: `docs/` directory +# Pipeline Guide + +- Always first write your "plan" in natural langage, then transcribe it in pipelex. +- You should ALWAYS RUN the terminal command `make validate` when you are writing a `.plx` file. It will ensure the pipe is runnable. If not, iterate. +- Please use POSIX standard for files. (enmpty lines, no trailing whitespaces, etc.) + +# Pipeline Structure Guide + +## Pipeline File Naming +- Files must be `.plx` for pipelines (Always add an empty line at the end of the file, and do not add trailing whitespaces to PLX files at all) +- Files must be `.py` for structures +- Use descriptive names in `snake_case` + +## Pipeline File Structure +A pipeline file has three main sections: +1. Domain statement +2. Concept definitions +3. Pipe definitions + +### Domain Statement +```plx +domain = "domain_name" +definition = "Description of the domain" # Optional +``` +Note: The domain name usually matches the plx filename for single-file domains. For multi-file domains, use the subdirectory name. + +### Concept Definitions +```plx +[concept] +ConceptName = "Description of the concept" # Should be the same name as the Structure ClassName you want to output +``` + +Important Rules: +- Use PascalCase for concept names +- Never use plurals (no "Stories", use "Story") +- Avoid adjectives (no "LargeText", use "Text") +- Don't redefine native concepts (Text, Image, PDF, TextAndImages, Number) +yes +### Pipe Definitions + +## Pipe Base Structure + +```plx +[pipe.your_pipe_name] +type = "PipeLLM" +definition = "A description of what your pipe does" +inputs = { input_1 = "ConceptName1", input_2 = "ConceptName2" } +output = "ConceptName" +``` + +DO NOT WRITE: +```plx +[pipe.your_pipe_name] +type = "pipe_sequence" +``` + +But it should be: + +```plx +[pipe.your_pipe_name] +type = "PipeSequence" +definition = "....." +``` + +The pipes will all have at least this base structure. +- `inputs`: Dictionnary of key behing the variable used in the prompts, and the value behing the ConceptName. It should ALSO LIST THE INPUTS OF THE INTERMEDIATE STEPS (if pipeSequence) or of the conditionnal pipes (if pipeCondition). +So If you have this error: +`StaticValidationError: missing_input_variable • domain='expense_validator' • pipe='validate_expense' • +variable='['ocr_input']'`` +That means that the pipe validate_expense is missing the input `ocr_input` because one of the subpipe is needing it. + +NEVER WRITE THE INPUTS BY BREAKING THE LINE LIKE THIS: + +```plx +inputs = { + input_1 = "ConceptName1", + input_2 = "ConceptName2" +} +``` + + +- `output`: The name of the concept to output. The `ConceptName` should have the same name as the python class if you want structured output: + +# Structured Models Rules + +## Model Location and Registration + +- Create models for structured generations related to "some_domain" in `pipelex_libraries/pipelines/.py` +- Models must inherit from `StructuredContent` or appropriate content type + +## Model Structure + +Concepts and their structure classes are meant to indicate an idea. +A Concept MUST NEVER be a plural noun and you should never create a SomeConceptList: lists and arrays are implicitly handled by Pipelex according to the context. Just define SomeConcept. + +**IMPORTANT: Never create unnecessary structure classes that only refine native concepts without adding fields.** + +DO NOT create structures like: +```python +class Joke(TextContent): + """A humorous text that makes people laugh.""" + pass +``` + +If a concept only refines a native concept (like Text, Image, etc.) without adding new fields, simply declare it in the .plx file: +```plx +[concept] +Joke = "A humorous text that makes people laugh." +``` +If you simply need to refine another native concept, construct it like this: +```plx +[concept.Landscape] +refines = "Image" +``` + +Only create a Python structure class when you need to add specific fields: + +```python +from datetime import datetime +from typing import List, Optional +from pydantic import Field + +from pipelex.core.stuffs.stuff_content import StructuredContent + +# IMPORTANT: THE CLASS MUST BE A SUBCLASS OF StructuredContent +class YourModel(StructuredContent): # Always be a subclass of StructuredContent + # Required fields + field1: str + field2: int + + # Optional fields with defaults + field3: Optional[str] = Field(None, "Description of field3") + field4: List[str] = Field(default_factory=list) + + # Date fields should remove timezone + date_field: Optional[datetime] = None +``` +## Usage + +Structures are meant to indicate what class to use for a particular Concept. In general they use the same name as the concept. + +Structure classes defined within `pipelex_libraries/pipelines/` are automatically loaded into the class_registry when setting up Pipelex, no need to do it manually. + + +## Best Practices for structures + +- Respect Pydantic v2 standards +- Use type hints for all fields +- Use `Field` declaration and write the description + + +## Pipe Controllers and Pipe Operator + +Look at the Pipes we have in order to adapt it. Pipes are organized in two categories: + +1. **Controllers** - For flow control: + - `PipeSequence` - For creating a sequence of multiple steps + - `PipeCondition` - If the next pipe depends of the expression of a stuff in the working memory + - `PipeParallel` - For parallelizing pipes + - `PipeBatch` - For running pipes in Batch over a ListContent + +2. **Operators** - For specific tasks: + - `PipeLLM` - Generate Text and Objects (include Vision LLM) + - `PipeOcr` - OCR Pipe + - `PipeImgGen` - Generate Images + - `PipeFunc` - For running classic python scripts + +# PipeSequence Guide + +## Purpose +PipeSequence executes multiple pipes in a defined order, where each step can use results from previous steps. + +## Basic Structure +```plx +[pipe.your_sequence_name] +type = "PipeSequence" +definition = "Description of what this sequence does" +inputs = { input_name = "InputType" } # All the inputs of the sub pipes, except the ones generated by intermediate steps +output = "OutputType" +steps = [ + { pipe = "first_pipe", result = "first_result" }, + { pipe = "second_pipe", result = "second_result" }, + { pipe = "final_pipe", result = "final_result" } +] +``` + +## Key Components + +1. **Steps Array**: List of pipes to execute in sequence + - `pipe`: Name of the pipe to execute + - `result`: Name to assign to the pipe's output that will be in the working memory + +## Using PipeBatch in Steps + +You can use PipeBatch functionality within steps using `batch_over` and `batch_as`: + +```plx +steps = [ + { pipe = "process_items", batch_over = "input_list", batch_as = "current_item", result = "processed_items" + } +] +``` + +1. **batch_over**: Specifies a `ListContent` field to iterate over. Each item in the list will be processed individually and IN PARALLEL by the pipe. + - Must be a `ListContent` type containing the items to process + - Can reference inputs or results from previous steps + +2. **batch_as**: Defines the name that will be used to reference the current item being processed + - This name can be used in the pipe's input mappings + - Makes each item from the batch available as a single element + +The result of a batched step will be a `ListContent` containing the outputs from processing each item. + +# PipeCondition Controller + +The PipeCondition controller allows you to implement conditional logic in your pipeline, choosing which pipe to execute based on an evaluated expression. It supports both direct expressions and expression templates. + +## Usage in PLX Configuration + +### Basic Usage with Direct Expression + +```plx +[pipe.conditional_operation] +type = "PipeCondition" +definition = "A conditonal pipe to decide wheter..." +inputs = { input_data = "CategoryInput" } +output = "native.Text" +expression = "input_data.category" + +[pipe.conditional_operation.pipe_map] +small = "process_small" +medium = "process_medium" +large = "process_large" +``` +or +```plx +[pipe.conditional_operation] +type = "PipeCondition" +definition = "A conditonal pipe to decide wheter..." +inputs = { input_data = "CategoryInput" } +output = "native.Text" +expression_template = "{{ input_data.category }}" # Jinja2 code + +[pipe.conditional_operation.pipe_map] +small = "process_small" +medium = "process_medium" +large = "process_large" +``` + +## Key Parameters + +- `expression`: Direct boolean or string expression (mutually exclusive with expression_template) +- `expression_template`: Jinja2 template for more complex conditional logic (mutually exclusive with expression) +- `pipe_map`: Dictionary mapping expression results to pipe codes : +1 - The key on the left (`small`, `medium`) is the result of `expression` or `expression_template`. +2 - The value on the right (`process_small`, `process_medium`, ..) is the name of the pipce to trigger + +# PipeBatch Controller + +The PipeBatch controller allows you to apply a pipe operation to each element in a list of inputs in parallele. It is created via a PipeSequence. + +## Usage in PLX Configuration + +```plx +[pipe.sequence_with_batch] +type = "PipeSequence" +definition = "A Sequence of pipes" +inputs = { input_data = "ConceptName" } +output = "OutputConceptName" +steps = [ + { pipe = "pipe_to_apply", batch_over = "input_list", batch_as = "current_item", result = "batch_results" } +] +``` + +## Key Parameters + +- `pipe`: The pipe operation to apply to each element in the batch +- `batch_over`: The name of the list in the context to iterate over +- `batch_as`: The name to use for the current element in the pipe's context +- `result`: Where to store the results of the batch operation + +# PipeLLM Guide + +## Purpose + +PipeLLM is used to: +1. Generate text or objects with LLMs +2. Process images with Vision LLMs + +## Basic Usage + +### Simple Text Generation +```plx +[pipe.write_story] +type = "PipeLLM" +definition = "Write a short story" +output = "Text" +prompt_template = """ +Write a short story about a programmer. +""" +``` + +### Structured Data Extraction +```plx +[pipe.extract_info] +type = "PipeLLM" +definition = "Extract information" +inputs = { text = "Text" } +output = "PersonInfo" +prompt_template = """ +Extract person information from this text: +@text +""" +``` + +### System Prompts +Add system-level instructions: +```plx +[pipe.expert_analysis] +type = "PipeLLM" +definition = "Expert analysis" +output = "Analysis" +system_prompt = "You are a data analysis expert" +prompt_template = "Analyze this data" +``` + +### Multiple Outputs +Generate multiple results: +```plx +[pipe.generate_ideas] +type = "PipeLLM" +definition = "Generate ideas" +output = "Idea" +nb_output = 3 # Generate exactly 3 ideas +# OR +multiple_output = true # Let the LLM decide how many to generate +``` + +### Vision Tasks +Process images with VLMs: +```plx +[pipe.analyze_image] +type = "PipeLLM" +definition = "Analyze image" +inputs = { image = "Image" } # `image` is the name of the stuff that contains the Image. If its in a stuff, you can add something like `{ "page.image": "Image" } +output = "ImageAnalysis" +prompt_template = "Describe what you see in this image" +``` + +# PipeOCR Guide + +## Purpose + +Extract text and images from an image or a PDF + +## Basic Usage + +### Simple Text Generation +```plx +[pipe.extract_info] +type = "PipeOcr" +definition = "extract the information" +inputs = { ocr_input = "PDF" } # or { ocr_input = "Image" } if its an image. This is the only input +output = "Page" +``` + +The input ALWAYS HAS TO BE `ocr_input` and the value is either of concept `Image` or `Pdf`. + +The output concept `Page` is a native concept, with the structure `PageContent`: +It corresponds to 1 page. Therefore, the PipeOcr is outputing a `ListContent` of `Page` + +```python +class TextAndImagesContent(StuffContent): + text: Optional[TextContent] + images: Optional[List[ImageContent]] + +class PageContent(StructuredContent): # CONCEPT IS "Page" + text_and_images: TextAndImagesContent + page_view: Optional[ImageContent] = None +``` +- `text_and_images` are the text, and the related images found in the input image or PDF. +- `page_view` is the screenshot of the whole pdf page/image. + +This rule explains how to write prompt templates in PipeLLM definitions. + +## Insert stuff inside a tagged block + +If the inserted text is supposedly long text, made of several lines or paragraphs, you want it inserted inside a block, possibly a block tagged and delimlited with proper syntax as one would do in a markdown documentation. To include stuff as a block, use the "@" prefix. + +Example template: +```plx +prompt_template = """ +Match the expense with its corresponding invoice: + +@expense + +@invoices +""" +``` +In this example, the expense data and the invoices data are obviously made of several lines each, that's why it makes sense to use the "@" prefix in order to have them delimited inside a block. Note that our preprocessor will automatically include the block's title, so it doens't need to be explictly written in the prompt template. + +**DO NOT write things like "Here is the expense: @expense".** +**DO write simply "@expense" alone in an isolated line.** + +## Insert stuff inline + +If the inserted text is short text and it makes sense to have it inserted directly into a sentence, you want it inserted inline. To insert stuff inline, use the "$" prefix. This will insert the stuff without delimiters and the content will be rendered as plain text. + +Example template: +```plx +prompt_template = """ +Your goal is to summarize everything related to $topic in the provided text: + +@text + +Please provide only the summary, with no additional text or explanations. +Your summary should not be longer than 2 sentences. +""" +``` + +Here, $topic will be inserted inline, whereas @text will be a a delimited block. +Be sure to make the proper choice of prefix for each insertion. + +**DO NOT write "$topic" alone in an isolated line.** +**DO write things like "Write an essay about $topic" included in an actual sentence.** + +# Example to execute a pipeline + +```python +import asyncio + +from pipelex import pretty_print +from pipelex.hub import get_pipeline_tracker, get_report_delegate +from pipelex.pipelex import Pipelex +from pipelex.pipeline.execute import execute_pipeline + +from cocode.pipelex_libraries.pipelines.examples.extract_gantt.gantt import GanttChart + +SAMPLE_NAME = "extract_gantt" +IMAGE_URL = "assets/gantt/gantt_tree_house.png" + + +async def extract_gantt(image_url: str) -> GanttChart: + # Run the pipe + pipe_output = await execute_pipeline( + pipe_code="extract_gantt_by_steps", + input_memory={ + "gantt_chart_image": { + "concept": "gantt.GanttImage", + "content": ImageContent(url=image_url), + } + }, + ) + # Output the result + return pipe_output.main_stuff_as(content_type=GanttChart) + + +# start Pipelex +Pipelex.make() + +# run sample using asyncio +gantt_chart = asyncio.run(extract_gantt(IMAGE_URL)) + +# Display cost report (tokens used and cost) +get_report_delegate().generate_report() +# output results +pretty_print(gantt_chart, title="Gantt Chart") +get_pipeline_tracker().output_flowchart() +``` + +The input memory is a dictionary of key-value pairs, where the key is the name of the input variable and the value provides details to make it a stuff object. The relevant definitions are: +```python +StuffContentOrData = Dict[str, Any] | StuffContent | List[Any] | str +ImplicitMemory = Dict[str, StuffContentOrData] +``` +As you can seen, we made it so different ways can be used to define that stuff using structured content or data. + +So here are a few concrete examples of calls to execute_pipeline with various ways to set up the input memory: + +```python +# Here we have a single input and it's a Text. +# If you assign a string, by default it will be considered as a TextContent. + pipe_output = await execute_pipeline( + pipe_code="master_advisory_orchestrator", + input_memory={ + "user_input": problem_description, + }, + ) + +# Here we have a single input and it's a PDF. +# Because PDFContent is a native concept, we can use it directly as a value, +# the system knows what content it corresponds to: + pipe_output = await execute_pipeline( + pipe_code="power_extractor_dpe", + input_memory={ + "ocr_input": PDFContent(url=pdf_url), + }, + ) + +# Here we have a single input and it's an Image. +# Because ImageContent is a native concept, we can use it directly as a value: + pipe_output = await execute_pipeline( + pipe_code="fashion_variation_pipeline", + input_memory={ + "fashion_photo": ImageContent(url=image_url), + }, + ) + +# Here we have a single input, it's an image but +# its actually a more specific concept gantt.GanttImage which refines Image, +# so we must provide it using a dict with the concept and the content: + pipe_output = await execute_pipeline( + pipe_code="extract_gantt_by_steps", + input_memory={ + "gantt_chart_image": { + "concept": "gantt.GanttImage", + "content": ImageContent(url=image_url), + } + }, + ) + +# Here is a more complex example with multiple inputs assigned using different ways: + pipe_output = await execute_pipeline( + pipe_code="retrieve_then_answer", + dynamic_output_concept_code="contracts.Fees", + input_memory={ + "text": load_text_from_path(path=text_path), + "question": { + "concept": "answer.Question", + "content": question, + }, + "client_instructions": client_instructions, + }, + ) +``` + +ALWAYS RUN `make validate` when you are finished writing pipelines: This checks for errors. If there are errors, iterate until it works. +Then, create an example file to run the pipeline in the `examples` folder. +But don't write documentation unless asked explicitly to. + +# Rules to choose LLM models used in PipeLLMs. + +## LLM Handles + +In order to use it in a pipe, an LLM is referenced by its llm_handle and possibly by an llm_preset. +Both llm_handles and llm_presets are defined in this toml config file: [base_llm_deck.toml](./cocode/pipelex_libraries/llm_deck/base_llm_deck.toml) + +## LLM Handles + +An llm_handle matches the handle (an id of sorts) with the full specification of the LLM to use, i.e.: +- llm_name +- llm_version +- llm_platform_choice + +The declaration of llm_handles looks like this in toml syntax: +```toml +[llm_handles] +gpt-4o-2024-11-20 = { llm_name = "gpt-4o", llm_version = "2024-11-20" } +``` + +In mosty cases, we only want to use version "latest" and llm_platform_choice "default" in which case the declaration is simply a match of the llm_handle to the llm_name, like this: +```toml +best-claude = "claude-4-opus" +best-gemini = "gemini-2.5-pro" +best-mistral = "mistral-large" +``` + +And of course, llm_handles are automatically assigned for all models by their name, with version "latest" and llm_platform_choice "default". + +## Using an LLM Handle in a PipeLLM + +Here is an example of using an llm_handle to specify which LLM to use in a PipeLLM: + +```plx +[pipe.hello_world] +type = "PipeLLM" +definition = "Write text about Hello World." +output = "Text" +llm = { llm_handle = "gpt-4o-mini", temperature = 0.9, max_tokens = "auto" } +prompt_template = """ +Write a haiku about Hello World. +""" +``` + +As you can see, to use the LLM, you must also indicate the temperature (float between 0 and 1) and max_tokens (either an int or the string "auto"). + +## LLM Presets + +Presets are meant to record the choice of an llm with its hyper parameters (temperature and max_tokens) if it's good for a particular task. LLM Presets are skill-oriented. + +Examples: +```toml +llm_to_reason = { llm_handle = "o4-mini", temperature = 1, max_tokens = "auto" } +llm_to_extract_invoice = { llm_handle = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" } +``` + +The interest is that these presets can be used to set the LLM choice in a PipeLLM, like this: + +```plx +[pipe.extract_invoice] +type = "PipeLLM" +definition = "Extract invoice information from an invoice text transcript" +inputs = { invoice_text = "InvoiceText" } +output = "Invoice" +llm = "llm_to_extract_invoice" +prompt_template = """ +Extract invoice information from this invoice: + +The category of this invoice is: $invoice_details.category. + +@invoice_text +""" +``` + +The setting here `llm = "llm_to_extract_invoice"` works because "llm_to_extract_invoice" has been declared as an llm_preset in the deck. +You must not use an LLM preset in a PipeLLM that does not exist in the deck. If needed, you can add llm presets. + + +You can override the predefined llm presets in [overrides.toml](./cocode/pipelex_libraries/llm_deck/overrides.toml). + +These rules apply when writing unit tests. - Always use pytest -### Test file structure +## Test file structure - Name test files with `test_` prefix - Use descriptive names that match the functionality being tested -- Place test files in the appropriate subdirectory of `tests/`: -- More precisely place async tests inside the `asynch` subdirectory +- Place test files in the appropriate test category directory: + - `tests/unit/` - for unit tests that test individual functions/classes in isolation + - `tests/integration/` - for integration tests that test component interactions + - `tests/e2e/` - for end-to-end tests that test complete workflows + - `tests/test_pipelines/` - for test pipeline definitions (PLX files and their structuring python files) +- Fixtures are defined in conftest.py modules at different levels of the hierarchy, their scope is handled by pytest +- Test data is placed inside test_data.py at different levels of the hierarchy, they must be imported with package paths from the root like `tests.pipelex.test_data`. Their content is all constants, regrouped inside classes to keep things tidy. +- Always put test inside Test classes. +- The pipelex pipelines should be stored in `tests/test_pipelines` as well as the related structured Output classes that inherit from `StructuredContent` -### Markers +## Markers Apply the appropriate markers: +- "llm: uses an LLM to generate text or objects" +- "imgg: uses an image generation AI" - "inference: uses either an LLM or an image generation AI" - "gha_disabled: will not be able to run properly on GitHub Actions" -- "codex_disabled: will not be able to run properly on Codex" # typically relevant for tests that need internet access, which Codex doesn't allow Several markers may be applied. For instance, if the test uses an LLM, then it uses inference, so you must mark with both `inference`and `llm`. -### Test Class Structure +## Tips + +- Never use the unittest.mock. Use pytest-mock + +## Test Class Structure Always group the tests of a module into a test class: ```python +@pytest.mark.llm @pytest.mark.inference @pytest.mark.asyncio(loop_scope="class") class TestFooBar: @@ -60,22 +796,131 @@ class TestFooBar: Sometimes it can be convenient to access the test's name in its body, for instance to include into a job_id. To achieve that, add the argument `request: FixtureRequest` into the signature and then you can get the test name using `cast(str, request.node.originalname), # type: ignore`. -## Linting & checking +# Pipe tests + +## Required imports for pipe tests + +```python +import pytest +from pytest import FixtureRequest +from pipelex import log, pretty_print +from pipelex.core.stuffs.stuff_factory import StuffBlueprint, StuffFactory +from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory +from pipelex.hub import get_report_delegate +from cocode.pipelex_libraries.pipelines.base_library.retrieve import RetrievedExcerpt +from pipelex.config_pipelex import get_config + +from pipelex.core.pipe import PipeAbstract, update_job_metadata_for_pipe +from pipelex.core.pipes.pipe_output import PipeOutput, PipeOutputType +from pipelex.core.pipes.pipe_run_params import PipeRunParams +from pipelex.core.pipes.pipe_run_params import PipeRunParams +from pipelex.pipe_works.pipe_router_protocol import PipeRouterProtocol +``` + +## Pipe test implementation steps + +1. Create Stuff from blueprint: + +```python +stuff = StuffFactory.make_stuff( + concept_code="RetrievedExcerpt", + domain="retrieve", + content=RetrievedExcerpt(text="", justification="") + name="retrieved_text", +) +``` + +2. Create Working Memory: + +```python +working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff) +``` + +3. Run the pipe: + +```python +pipe_output: PipeOutput = await pipe_router.run_pipe( + pipe_code="pipe_name", + pipe_run_params=PipeRunParamsFactory.make_run_params(), + working_memory=working_memory, + job_metadata=JobMetadata(), +) +``` + +4. Log output and generate report: + +```python +pretty_print(pipe_output, title=f"Pipe output") +get_report_delegate().generate_report() +``` + +5. Basic assertions: + +```python +assert pipe_output is not None +assert pipe_output.working_memory is not None +assert pipe_output.main_stuff is not None +``` + +## Test Data Organization + +- If it's not already there, create a `test_data.py` file in the test directory +- Define test cases using `StuffBlueprint`: + +```python +class TestCases: + CASE_BLUEPRINT_1 = StuffBlueprint( + name="test_case_1", + concept_code="domain.ConceptName1", + value="test_value" + ) + CASE_BLUEPRINT_2 = StuffBlueprint( + name="test_case_2", + concept_code="domain.ConceptName2", + value="test_value" + ) + + CASE_BLUEPRINTS: ClassVar[List[Tuple[str, str]]] = [ # topic, blueprint" + ("topic1", CASE_BLUEPRINT_1), + ("topic2", CASE_BLUEPRINT_2), + ] +``` + +Note how we avoid initializing a default mutable value within a class instance, instead we use ClassVar. +Also note that we provide a topic for the test case, which is purely for convenience. + +## Best Practices for Testing + +- Use parametrize for multiple test cases +- Test both success and failure cases +- Verify working memory state +- Check output structure and content +- Use meaningful test case names +- Include docstrings explaining test purpose +- Log outputs for debugging +- Generate reports for cost tracking + +# Test-Driven Development Guide + +This document outlines our test-driven development (TDD) process and the tools available for testing. + +## TDD Cycle + +1. **Write a Test First** +[pytest.mdc](pytest.mdc) + +2. **Write the Code** + - Implement the minimum amount of code needed to pass the test + - Follow the project's coding standards + - Keep it simple - don't write more than needed -- Run `make lint` -> it runs `ruff check . --fix` to enforce all our linting rules -- Run `make pyright` -> it typechecks with pyright using proper settings -- Run `make mypy` -> it typechecks with mypy using proper settings - - if you added a dependency and mypy complains that it's not typed, add it to the list of modules in [[tool.mypy.overrides]] in pyproject.toml, be sure to signal it in your PR recap so that maintainers can look for existing stubs +3. **Run Linting and Type Checking** +[coding_standards.mdc](coding_standards.mdc) -## Testing +4. **Refactor if needed** +If the code needs refactoring, with the best practices [coding_standards.mdc](coding_standards.mdc) -- Always test with `make codex-tests` -> it runs pytest on our `tests/` directory using proper settings -- If all unit tests pass, run `make validate` -> it runs a minimal version of our app with just the inits and data loading +5. **Validate tests** -## PR Instructions +Remember: The key to TDD is writing the test first and letting it drive your implementation. Always run the full test suite and quality checks before considering a feature complete. -- Run `make fix-unused-imports` -> removes unused imports, required to validate PR -- Re-run checks in one call with `make check` -> formatting and linting with Ruff, type-checking with Pyright and Mypy -- Re-run `make codex-tests` -- Write a one-line summary of the changes. -- Be sure to list changes made to configs, tests and dependencies diff --git a/CLAUDE.md b/CLAUDE.md index 8b3d8a7..993a025 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,19 +1,745 @@ -# General rules +Concatenation +# Coding Standards & Best Practices -## Repo structure +This document outlines the core coding standards, best practices, and quality control procedures for the codebase. -Pipelex is a framework to run low-code AI workflows for repeatable processes. -This python >=3.10 code is in the `pipelex` directory. +## Type Hints -## Code Style & formatting +1. **Always Use Type Hints** + - Every function parameter must be typed + - Every function return must be typed + - Use type hints for all variables where type is not obvious + - Use types with Uppercase first letter (Dict[], List[], etc.) -- Imitate existing style -- Use type hints -- Respect Pydantic v2 standard -- Use Typer for CLIs -- Use explicit keyword arguments for function calls with multiple parameters (e.g., `func(arg_name=value)` not just `func(value)`) -- Add trailing commas to multi-line lists, dicts, function arguments, and tuples with >2 items (helps with cleaner diffs and prevents syntax errors when adding items) -- All imports inside this repo's packages must be absolute package paths from the root +2. **StrEnum** + - Import from `pipelex.types`: + ```python + from pipelex.types import StrEnum + ``` + +## BaseModel Standards + +- Respect Pydantic v2 standards +- Keep models focused and single-purpose +- Use descriptive field names +- Use type hints for all fields +- Document complex validations +- Use Optional[] for nullable fields +- Use Field(default_factory=...) for mutable defaults + +## Factory Pattern + +- Use Factory Pattern for object creation when dealing with multiple implementations + +## Documentation + +1. **Docstring Format** + ```python + def process_image(image_path: str, size: Tuple[int, int]) -> bytes: + """Process and resize an image. + + Args: + image_path: Path to the source image + size: Tuple of (width, height) for resizing + + Returns: + Processed image as bytes + """ + pass + ``` + +2. **Class Documentation** + ```python + class ImageProcessor: + """Handles image processing operations. + + Provides methods for resizing, converting, and optimizing images. + """ + ``` + +## Error Handling + +1. **Graceful Error Handling** + - Use try/except blocks with specific exceptions + - Convert third-party exceptions to custom ones + ```python + try: + from fal_client import AsyncClient as FalAsyncClient + except ImportError as exc: + raise MissingDependencyError( + "fal-client", "fal", + "The fal-client SDK is required to use FAL models." + ) from exc + ``` + +## Code Quality Checks + +### Linting and Type Checking + +Before finalizing a task, run: +```bash +make fix-unused-imports +make check +``` + +This runs multiple code quality tools: +- Pyright: Static type checking +- Ruff: Fast Python linter +- Mypy: Static type checker + +Always fix any issues reported by these tools before proceeding. + +### Running Tests + +1. **Quick Test Run** (no LLM/image generation): + ```bash + make tp + ``` + Runs tests with markers: `(dry_runnable or not (inference or llm or imgg or ocr)) and not (needs_output or pipelex_api)` + +2. **Specific Tests**: + ```bash + make tp TEST=TestClassName + # or + make tp TEST=test_function_name + ``` + Note: Matches names starting with the provided string. + +**Important**: Never run `make ti`, `make test-inference`, `make to`, `make test-ocr`, `make tg`, or `make test-imgg` - these use costly inference. + +## Pipelines + +- All pipeline definitions go in `cocode/pipelex_libraries/pipelines/` +- Always validate pipelines after creation/edit with `make validate`. + Iterate if there are errors. + +## Project Structure + +- **Pipelines**: `cocode/pipelex_libraries/pipelines/` +- **Tests**: `tests/` directory +- **Documentation**: `docs/` directory +# Pipeline Guide + +- Always first write your "plan" in natural langage, then transcribe it in pipelex. +- You should ALWAYS RUN the terminal command `make validate` when you are writing a `.plx` file. It will ensure the pipe is runnable. If not, iterate. +- Please use POSIX standard for files. (enmpty lines, no trailing whitespaces, etc.) + +# Pipeline Structure Guide + +## Pipeline File Naming +- Files must be `.plx` for pipelines (Always add an empty line at the end of the file, and do not add trailing whitespaces to PLX files at all) +- Files must be `.py` for structures +- Use descriptive names in `snake_case` + +## Pipeline File Structure +A pipeline file has three main sections: +1. Domain statement +2. Concept definitions +3. Pipe definitions + +### Domain Statement +```plx +domain = "domain_name" +definition = "Description of the domain" # Optional +``` +Note: The domain name usually matches the plx filename for single-file domains. For multi-file domains, use the subdirectory name. + +### Concept Definitions +```plx +[concept] +ConceptName = "Description of the concept" # Should be the same name as the Structure ClassName you want to output +``` + +Important Rules: +- Use PascalCase for concept names +- Never use plurals (no "Stories", use "Story") +- Avoid adjectives (no "LargeText", use "Text") +- Don't redefine native concepts (Text, Image, PDF, TextAndImages, Number) +yes +### Pipe Definitions + +## Pipe Base Structure + +```plx +[pipe.your_pipe_name] +type = "PipeLLM" +definition = "A description of what your pipe does" +inputs = { input_1 = "ConceptName1", input_2 = "ConceptName2" } +output = "ConceptName" +``` + +DO NOT WRITE: +```plx +[pipe.your_pipe_name] +type = "pipe_sequence" +``` + +But it should be: + +```plx +[pipe.your_pipe_name] +type = "PipeSequence" +definition = "....." +``` + +The pipes will all have at least this base structure. +- `inputs`: Dictionnary of key behing the variable used in the prompts, and the value behing the ConceptName. It should ALSO LIST THE INPUTS OF THE INTERMEDIATE STEPS (if pipeSequence) or of the conditionnal pipes (if pipeCondition). +So If you have this error: +`StaticValidationError: missing_input_variable • domain='expense_validator' • pipe='validate_expense' • +variable='['ocr_input']'`` +That means that the pipe validate_expense is missing the input `ocr_input` because one of the subpipe is needing it. + +NEVER WRITE THE INPUTS BY BREAKING THE LINE LIKE THIS: + +```plx +inputs = { + input_1 = "ConceptName1", + input_2 = "ConceptName2" +} +``` + + +- `output`: The name of the concept to output. The `ConceptName` should have the same name as the python class if you want structured output: + +# Structured Models Rules + +## Model Location and Registration + +- Create models for structured generations related to "some_domain" in `pipelex_libraries/pipelines/.py` +- Models must inherit from `StructuredContent` or appropriate content type + +## Model Structure + +Concepts and their structure classes are meant to indicate an idea. +A Concept MUST NEVER be a plural noun and you should never create a SomeConceptList: lists and arrays are implicitly handled by Pipelex according to the context. Just define SomeConcept. + +**IMPORTANT: Never create unnecessary structure classes that only refine native concepts without adding fields.** + +DO NOT create structures like: +```python +class Joke(TextContent): + """A humorous text that makes people laugh.""" + pass +``` + +If a concept only refines a native concept (like Text, Image, etc.) without adding new fields, simply declare it in the .plx file: +```plx +[concept] +Joke = "A humorous text that makes people laugh." +``` +If you simply need to refine another native concept, construct it like this: +```plx +[concept.Landscape] +refines = "Image" +``` + +Only create a Python structure class when you need to add specific fields: + +```python +from datetime import datetime +from typing import List, Optional +from pydantic import Field + +from pipelex.core.stuffs.stuff_content import StructuredContent + +# IMPORTANT: THE CLASS MUST BE A SUBCLASS OF StructuredContent +class YourModel(StructuredContent): # Always be a subclass of StructuredContent + # Required fields + field1: str + field2: int + + # Optional fields with defaults + field3: Optional[str] = Field(None, "Description of field3") + field4: List[str] = Field(default_factory=list) + + # Date fields should remove timezone + date_field: Optional[datetime] = None +``` +## Usage + +Structures are meant to indicate what class to use for a particular Concept. In general they use the same name as the concept. + +Structure classes defined within `pipelex_libraries/pipelines/` are automatically loaded into the class_registry when setting up Pipelex, no need to do it manually. + + +## Best Practices for structures + +- Respect Pydantic v2 standards +- Use type hints for all fields +- Use `Field` declaration and write the description + + +## Pipe Controllers and Pipe Operator + +Look at the Pipes we have in order to adapt it. Pipes are organized in two categories: + +1. **Controllers** - For flow control: + - `PipeSequence` - For creating a sequence of multiple steps + - `PipeCondition` - If the next pipe depends of the expression of a stuff in the working memory + - `PipeParallel` - For parallelizing pipes + - `PipeBatch` - For running pipes in Batch over a ListContent + +2. **Operators** - For specific tasks: + - `PipeLLM` - Generate Text and Objects (include Vision LLM) + - `PipeOcr` - OCR Pipe + - `PipeImgGen` - Generate Images + - `PipeFunc` - For running classic python scripts + +# PipeSequence Guide + +## Purpose +PipeSequence executes multiple pipes in a defined order, where each step can use results from previous steps. + +## Basic Structure +```plx +[pipe.your_sequence_name] +type = "PipeSequence" +definition = "Description of what this sequence does" +inputs = { input_name = "InputType" } # All the inputs of the sub pipes, except the ones generated by intermediate steps +output = "OutputType" +steps = [ + { pipe = "first_pipe", result = "first_result" }, + { pipe = "second_pipe", result = "second_result" }, + { pipe = "final_pipe", result = "final_result" } +] +``` + +## Key Components + +1. **Steps Array**: List of pipes to execute in sequence + - `pipe`: Name of the pipe to execute + - `result`: Name to assign to the pipe's output that will be in the working memory + +## Using PipeBatch in Steps + +You can use PipeBatch functionality within steps using `batch_over` and `batch_as`: + +```plx +steps = [ + { pipe = "process_items", batch_over = "input_list", batch_as = "current_item", result = "processed_items" + } +] +``` + +1. **batch_over**: Specifies a `ListContent` field to iterate over. Each item in the list will be processed individually and IN PARALLEL by the pipe. + - Must be a `ListContent` type containing the items to process + - Can reference inputs or results from previous steps + +2. **batch_as**: Defines the name that will be used to reference the current item being processed + - This name can be used in the pipe's input mappings + - Makes each item from the batch available as a single element + +The result of a batched step will be a `ListContent` containing the outputs from processing each item. + +# PipeCondition Controller + +The PipeCondition controller allows you to implement conditional logic in your pipeline, choosing which pipe to execute based on an evaluated expression. It supports both direct expressions and expression templates. + +## Usage in PLX Configuration + +### Basic Usage with Direct Expression + +```plx +[pipe.conditional_operation] +type = "PipeCondition" +definition = "A conditonal pipe to decide wheter..." +inputs = { input_data = "CategoryInput" } +output = "native.Text" +expression = "input_data.category" + +[pipe.conditional_operation.pipe_map] +small = "process_small" +medium = "process_medium" +large = "process_large" +``` +or +```plx +[pipe.conditional_operation] +type = "PipeCondition" +definition = "A conditonal pipe to decide wheter..." +inputs = { input_data = "CategoryInput" } +output = "native.Text" +expression_template = "{{ input_data.category }}" # Jinja2 code + +[pipe.conditional_operation.pipe_map] +small = "process_small" +medium = "process_medium" +large = "process_large" +``` + +## Key Parameters + +- `expression`: Direct boolean or string expression (mutually exclusive with expression_template) +- `expression_template`: Jinja2 template for more complex conditional logic (mutually exclusive with expression) +- `pipe_map`: Dictionary mapping expression results to pipe codes : +1 - The key on the left (`small`, `medium`) is the result of `expression` or `expression_template`. +2 - The value on the right (`process_small`, `process_medium`, ..) is the name of the pipce to trigger + +# PipeBatch Controller + +The PipeBatch controller allows you to apply a pipe operation to each element in a list of inputs in parallele. It is created via a PipeSequence. + +## Usage in PLX Configuration + +```plx +[pipe.sequence_with_batch] +type = "PipeSequence" +definition = "A Sequence of pipes" +inputs = { input_data = "ConceptName" } +output = "OutputConceptName" +steps = [ + { pipe = "pipe_to_apply", batch_over = "input_list", batch_as = "current_item", result = "batch_results" } +] +``` + +## Key Parameters + +- `pipe`: The pipe operation to apply to each element in the batch +- `batch_over`: The name of the list in the context to iterate over +- `batch_as`: The name to use for the current element in the pipe's context +- `result`: Where to store the results of the batch operation + +# PipeLLM Guide + +## Purpose + +PipeLLM is used to: +1. Generate text or objects with LLMs +2. Process images with Vision LLMs + +## Basic Usage + +### Simple Text Generation +```plx +[pipe.write_story] +type = "PipeLLM" +definition = "Write a short story" +output = "Text" +prompt_template = """ +Write a short story about a programmer. +""" +``` + +### Structured Data Extraction +```plx +[pipe.extract_info] +type = "PipeLLM" +definition = "Extract information" +inputs = { text = "Text" } +output = "PersonInfo" +prompt_template = """ +Extract person information from this text: +@text +""" +``` + +### System Prompts +Add system-level instructions: +```plx +[pipe.expert_analysis] +type = "PipeLLM" +definition = "Expert analysis" +output = "Analysis" +system_prompt = "You are a data analysis expert" +prompt_template = "Analyze this data" +``` + +### Multiple Outputs +Generate multiple results: +```plx +[pipe.generate_ideas] +type = "PipeLLM" +definition = "Generate ideas" +output = "Idea" +nb_output = 3 # Generate exactly 3 ideas +# OR +multiple_output = true # Let the LLM decide how many to generate +``` + +### Vision Tasks +Process images with VLMs: +```plx +[pipe.analyze_image] +type = "PipeLLM" +definition = "Analyze image" +inputs = { image = "Image" } # `image` is the name of the stuff that contains the Image. If its in a stuff, you can add something like `{ "page.image": "Image" } +output = "ImageAnalysis" +prompt_template = "Describe what you see in this image" +``` + +# PipeOCR Guide + +## Purpose + +Extract text and images from an image or a PDF + +## Basic Usage + +### Simple Text Generation +```plx +[pipe.extract_info] +type = "PipeOcr" +definition = "extract the information" +inputs = { ocr_input = "PDF" } # or { ocr_input = "Image" } if its an image. This is the only input +output = "Page" +``` + +The input ALWAYS HAS TO BE `ocr_input` and the value is either of concept `Image` or `Pdf`. + +The output concept `Page` is a native concept, with the structure `PageContent`: +It corresponds to 1 page. Therefore, the PipeOcr is outputing a `ListContent` of `Page` + +```python +class TextAndImagesContent(StuffContent): + text: Optional[TextContent] + images: Optional[List[ImageContent]] + +class PageContent(StructuredContent): # CONCEPT IS "Page" + text_and_images: TextAndImagesContent + page_view: Optional[ImageContent] = None +``` +- `text_and_images` are the text, and the related images found in the input image or PDF. +- `page_view` is the screenshot of the whole pdf page/image. + +This rule explains how to write prompt templates in PipeLLM definitions. + +## Insert stuff inside a tagged block + +If the inserted text is supposedly long text, made of several lines or paragraphs, you want it inserted inside a block, possibly a block tagged and delimlited with proper syntax as one would do in a markdown documentation. To include stuff as a block, use the "@" prefix. + +Example template: +```plx +prompt_template = """ +Match the expense with its corresponding invoice: + +@expense + +@invoices +""" +``` +In this example, the expense data and the invoices data are obviously made of several lines each, that's why it makes sense to use the "@" prefix in order to have them delimited inside a block. Note that our preprocessor will automatically include the block's title, so it doens't need to be explictly written in the prompt template. + +**DO NOT write things like "Here is the expense: @expense".** +**DO write simply "@expense" alone in an isolated line.** + +## Insert stuff inline + +If the inserted text is short text and it makes sense to have it inserted directly into a sentence, you want it inserted inline. To insert stuff inline, use the "$" prefix. This will insert the stuff without delimiters and the content will be rendered as plain text. + +Example template: +```plx +prompt_template = """ +Your goal is to summarize everything related to $topic in the provided text: + +@text + +Please provide only the summary, with no additional text or explanations. +Your summary should not be longer than 2 sentences. +""" +``` + +Here, $topic will be inserted inline, whereas @text will be a a delimited block. +Be sure to make the proper choice of prefix for each insertion. + +**DO NOT write "$topic" alone in an isolated line.** +**DO write things like "Write an essay about $topic" included in an actual sentence.** + +# Example to execute a pipeline + +```python +import asyncio + +from pipelex import pretty_print +from pipelex.hub import get_pipeline_tracker, get_report_delegate +from pipelex.pipelex import Pipelex +from pipelex.pipeline.execute import execute_pipeline + +from cocode.pipelex_libraries.pipelines.examples.extract_gantt.gantt import GanttChart + +SAMPLE_NAME = "extract_gantt" +IMAGE_URL = "assets/gantt/gantt_tree_house.png" + + +async def extract_gantt(image_url: str) -> GanttChart: + # Run the pipe + pipe_output = await execute_pipeline( + pipe_code="extract_gantt_by_steps", + input_memory={ + "gantt_chart_image": { + "concept": "gantt.GanttImage", + "content": ImageContent(url=image_url), + } + }, + ) + # Output the result + return pipe_output.main_stuff_as(content_type=GanttChart) + + +# start Pipelex +Pipelex.make() + +# run sample using asyncio +gantt_chart = asyncio.run(extract_gantt(IMAGE_URL)) + +# Display cost report (tokens used and cost) +get_report_delegate().generate_report() +# output results +pretty_print(gantt_chart, title="Gantt Chart") +get_pipeline_tracker().output_flowchart() +``` + +The input memory is a dictionary of key-value pairs, where the key is the name of the input variable and the value provides details to make it a stuff object. The relevant definitions are: +```python +StuffContentOrData = Dict[str, Any] | StuffContent | List[Any] | str +ImplicitMemory = Dict[str, StuffContentOrData] +``` +As you can seen, we made it so different ways can be used to define that stuff using structured content or data. + +So here are a few concrete examples of calls to execute_pipeline with various ways to set up the input memory: + +```python +# Here we have a single input and it's a Text. +# If you assign a string, by default it will be considered as a TextContent. + pipe_output = await execute_pipeline( + pipe_code="master_advisory_orchestrator", + input_memory={ + "user_input": problem_description, + }, + ) + +# Here we have a single input and it's a PDF. +# Because PDFContent is a native concept, we can use it directly as a value, +# the system knows what content it corresponds to: + pipe_output = await execute_pipeline( + pipe_code="power_extractor_dpe", + input_memory={ + "ocr_input": PDFContent(url=pdf_url), + }, + ) + +# Here we have a single input and it's an Image. +# Because ImageContent is a native concept, we can use it directly as a value: + pipe_output = await execute_pipeline( + pipe_code="fashion_variation_pipeline", + input_memory={ + "fashion_photo": ImageContent(url=image_url), + }, + ) + +# Here we have a single input, it's an image but +# its actually a more specific concept gantt.GanttImage which refines Image, +# so we must provide it using a dict with the concept and the content: + pipe_output = await execute_pipeline( + pipe_code="extract_gantt_by_steps", + input_memory={ + "gantt_chart_image": { + "concept": "gantt.GanttImage", + "content": ImageContent(url=image_url), + } + }, + ) + +# Here is a more complex example with multiple inputs assigned using different ways: + pipe_output = await execute_pipeline( + pipe_code="retrieve_then_answer", + dynamic_output_concept_code="contracts.Fees", + input_memory={ + "text": load_text_from_path(path=text_path), + "question": { + "concept": "answer.Question", + "content": question, + }, + "client_instructions": client_instructions, + }, + ) +``` + +ALWAYS RUN `make validate` when you are finished writing pipelines: This checks for errors. If there are errors, iterate until it works. +Then, create an example file to run the pipeline in the `examples` folder. +But don't write documentation unless asked explicitly to. + +# Rules to choose LLM models used in PipeLLMs. + +## LLM Handles + +In order to use it in a pipe, an LLM is referenced by its llm_handle and possibly by an llm_preset. +Both llm_handles and llm_presets are defined in this toml config file: [base_llm_deck.toml](./cocode/pipelex_libraries/llm_deck/base_llm_deck.toml) + +## LLM Handles + +An llm_handle matches the handle (an id of sorts) with the full specification of the LLM to use, i.e.: +- llm_name +- llm_version +- llm_platform_choice + +The declaration of llm_handles looks like this in toml syntax: +```toml +[llm_handles] +gpt-4o-2024-11-20 = { llm_name = "gpt-4o", llm_version = "2024-11-20" } +``` + +In mosty cases, we only want to use version "latest" and llm_platform_choice "default" in which case the declaration is simply a match of the llm_handle to the llm_name, like this: +```toml +best-claude = "claude-4-opus" +best-gemini = "gemini-2.5-pro" +best-mistral = "mistral-large" +``` + +And of course, llm_handles are automatically assigned for all models by their name, with version "latest" and llm_platform_choice "default". + +## Using an LLM Handle in a PipeLLM + +Here is an example of using an llm_handle to specify which LLM to use in a PipeLLM: + +```plx +[pipe.hello_world] +type = "PipeLLM" +definition = "Write text about Hello World." +output = "Text" +llm = { llm_handle = "gpt-4o-mini", temperature = 0.9, max_tokens = "auto" } +prompt_template = """ +Write a haiku about Hello World. +""" +``` + +As you can see, to use the LLM, you must also indicate the temperature (float between 0 and 1) and max_tokens (either an int or the string "auto"). + +## LLM Presets + +Presets are meant to record the choice of an llm with its hyper parameters (temperature and max_tokens) if it's good for a particular task. LLM Presets are skill-oriented. + +Examples: +```toml +llm_to_reason = { llm_handle = "o4-mini", temperature = 1, max_tokens = "auto" } +llm_to_extract_invoice = { llm_handle = "claude-3-7-sonnet", temperature = 0.1, max_tokens = "auto" } +``` + +The interest is that these presets can be used to set the LLM choice in a PipeLLM, like this: + +```plx +[pipe.extract_invoice] +type = "PipeLLM" +definition = "Extract invoice information from an invoice text transcript" +inputs = { invoice_text = "InvoiceText" } +output = "Invoice" +llm = "llm_to_extract_invoice" +prompt_template = """ +Extract invoice information from this invoice: + +The category of this invoice is: $invoice_details.category. + +@invoice_text +""" +``` + +The setting here `llm = "llm_to_extract_invoice"` works because "llm_to_extract_invoice" has been declared as an llm_preset in the deck. +You must not use an LLM preset in a PipeLLM that does not exist in the deck. If needed, you can add llm presets. + + +You can override the predefined llm presets in [overrides.toml](./cocode/pipelex_libraries/llm_deck/overrides.toml). + +These rules apply when writing unit tests. +- Always use pytest ## Test file structure @@ -28,7 +754,6 @@ This python >=3.10 code is in the `pipelex` directory. - Test data is placed inside test_data.py at different levels of the hierarchy, they must be imported with package paths from the root like `tests.pipelex.test_data`. Their content is all constants, regrouped inside classes to keep things tidy. - Always put test inside Test classes. - The pipelex pipelines should be stored in `tests/test_pipelines` as well as the related structured Output classes that inherit from `StructuredContent` -- Never use unittest.mock. Use pytest-mock. ## Markers @@ -40,6 +765,10 @@ Apply the appropriate markers: Several markers may be applied. For instance, if the test uses an LLM, then it uses inference, so you must mark with both `inference`and `llm`. +## Tips + +- Never use the unittest.mock. Use pytest-mock + ## Test Class Structure Always group the tests of a module into a test class: @@ -65,28 +794,133 @@ class TestFooBar: # Test implementation ``` -## Linting & checking +Sometimes it can be convenient to access the test's name in its body, for instance to include into a job_id. To achieve that, add the argument `request: FixtureRequest` into the signature and then you can get the test name using `cast(str, request.node.originalname), # type: ignore`. + +# Pipe tests + +## Required imports for pipe tests + +```python +import pytest +from pytest import FixtureRequest +from pipelex import log, pretty_print +from pipelex.core.stuffs.stuff_factory import StuffBlueprint, StuffFactory +from pipelex.core.memory.working_memory_factory import WorkingMemoryFactory +from pipelex.hub import get_report_delegate +from cocode.pipelex_libraries.pipelines.base_library.retrieve import RetrievedExcerpt +from pipelex.config_pipelex import get_config + +from pipelex.core.pipe import PipeAbstract, update_job_metadata_for_pipe +from pipelex.core.pipes.pipe_output import PipeOutput, PipeOutputType +from pipelex.core.pipes.pipe_run_params import PipeRunParams +from pipelex.core.pipes.pipe_run_params import PipeRunParams +from pipelex.pipe_works.pipe_router_protocol import PipeRouterProtocol +``` + +## Pipe test implementation steps + +1. Create Stuff from blueprint: + +```python +stuff = StuffFactory.make_stuff( + concept_code="RetrievedExcerpt", + domain="retrieve", + content=RetrievedExcerpt(text="", justification="") + name="retrieved_text", +) +``` + +2. Create Working Memory: + +```python +working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff) +``` + +3. Run the pipe: + +```python +pipe_output: PipeOutput = await pipe_router.run_pipe( + pipe_code="pipe_name", + pipe_run_params=PipeRunParamsFactory.make_run_params(), + working_memory=working_memory, + job_metadata=JobMetadata(), +) +``` + +4. Log output and generate report: + +```python +pretty_print(pipe_output, title=f"Pipe output") +get_report_delegate().generate_report() +``` + +5. Basic assertions: + +```python +assert pipe_output is not None +assert pipe_output.working_memory is not None +assert pipe_output.main_stuff is not None +``` + +## Test Data Organization + +- If it's not already there, create a `test_data.py` file in the test directory +- Define test cases using `StuffBlueprint`: + +```python +class TestCases: + CASE_BLUEPRINT_1 = StuffBlueprint( + name="test_case_1", + concept_code="domain.ConceptName1", + value="test_value" + ) + CASE_BLUEPRINT_2 = StuffBlueprint( + name="test_case_2", + concept_code="domain.ConceptName2", + value="test_value" + ) + + CASE_BLUEPRINTS: ClassVar[List[Tuple[str, str]]] = [ # topic, blueprint" + ("topic1", CASE_BLUEPRINT_1), + ("topic2", CASE_BLUEPRINT_2), + ] +``` + +Note how we avoid initializing a default mutable value within a class instance, instead we use ClassVar. +Also note that we provide a topic for the test case, which is purely for convenience. + +## Best Practices for Testing + +- Use parametrize for multiple test cases +- Test both success and failure cases +- Verify working memory state +- Check output structure and content +- Use meaningful test case names +- Include docstrings explaining test purpose +- Log outputs for debugging +- Generate reports for cost tracking + +# Test-Driven Development Guide + +This document outlines our test-driven development (TDD) process and the tools available for testing. + +## TDD Cycle -- Run `make lint` -> it runs `ruff check . --fix` to enforce all our linting rules -- Run `make pyright` -> it typechecks with pyright using proper settings -- Run `make mypy` -> it typechecks with mypy using proper settings - - if you added a dependency and mypy complains that it's not typed, add it to the list of modules in [[tool.mypy.overrides]] in pyproject.toml, be sure to signal it in your PR recap so that maintainers can look for existing stubs -- After `make pyright`, you must also check with `make mypy` +1. **Write a Test First** +[pytest.mdc](pytest.mdc) -## Testing +2. **Write the Code** + - Implement the minimum amount of code needed to pass the test + - Follow the project's coding standards + - Keep it simple - don't write more than needed -- Always test with `make t` -> it runs pytest using proper settings -- If some pytest tests fail, run pytest on the failed ones with the required verbosity to diagnose the issue -- If all unit tests pass, run `make validate` -> it runs a minimal version of our app with just the inits and data loading +3. **Run Linting and Type Checking** +[coding_standards.mdc](coding_standards.mdc) -## PR Instructions +4. **Refactor if needed** +If the code needs refactoring, with the best practices [coding_standards.mdc](coding_standards.mdc) -- Run `make fix-unused-imports` -> removes unused imports, required to validate PR -- Re-run checks in one call with `make check` -> formatting and linting with Ruff, type-checking with Pyright and Mypy -- Re-run `make codex-tests` -- Write a one-line summary of the changes. -- Be sure to list changes made to configs, tests and dependencies +5. **Validate tests** -## More docs +Remember: The key to TDD is writing the test first and letting it drive your implementation. Always run the full test suite and quality checks before considering a feature complete. -- Scan the *.mdc files in .cursor/rules/ to get usefull details and explanations on the codebase \ No newline at end of file diff --git a/CLI_README.md b/CLI_README.md index 83436f6..e923b3f 100644 --- a/CLI_README.md +++ b/CLI_README.md @@ -340,7 +340,7 @@ cocode validate Display comprehensive help and examples. ```bash -cocode help +cocode --help ``` ## Python Processing Rules @@ -416,7 +416,7 @@ cocode github auth 1. **Validate Setup**: Run `cocode validate` to ensure everything is configured correctly 2. **Basic Analysis**: Try `cocode repox` to analyze your current directory 3. **Explore Pipelines**: Use `cocode swe-from-repo --dry` to test SWE pipelines -4. **Get Help**: Use `cocode help` for detailed examples and ` --help` for specific options +4. **Get Help**: Use `cocode --help` for command overview and ` --help` for specific options ## Examples by Use Case diff --git a/docs/pages/commands.md b/docs/pages/commands.md index 37c04f2..99d435b 100644 --- a/docs/pages/commands.md +++ b/docs/pages/commands.md @@ -269,19 +269,6 @@ cocode github sync-labels [OPTIONS] REPO LABELS_FILE - `--dry-run` - Show what would be done without making changes - `--delete-extra` - Delete labels not in the standard set -## show-pipe - -Show pipe definition from the pipe library. - -```bash -cocode show-pipe PIPE_CODE -``` - -**Arguments:** -- `PIPE_CODE` - Pipeline code to show definition for - -Displays the complete pipe definition including configuration, steps, and metadata from the pipe library. - ## Other commands - `cocode validate` - Check setup From 2c8b32831c61d4c36318f04fe2b0acf2546faebc Mon Sep 17 00:00:00 2001 From: Louis Choquel Date: Sat, 6 Sep 2025 00:21:36 +0200 Subject: [PATCH 05/10] Better-extension-and-Blackbox-support (#42) --- .vscode/extensions.json | 7 +++++++ .vscode/settings.json | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 .vscode/extensions.json diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..d95081f --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,7 @@ +{ + "recommendations": [ + "Pipelex.pipelex", + "charliermarsh.ruff", + "matangover.mypy" + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index f06faf6..e908f73 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -19,5 +19,9 @@ "tests" ], "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true + "python.testing.pytestEnabled": true, + "djlint.showInstallError": false, + "files.associations": { + "*.plx": "plx" + } } \ No newline at end of file From 8735c54fc6ec7a103b83e8e4fd1de7111e1ef5fc Mon Sep 17 00:00:00 2001 From: Louis Choquel Date: Sat, 6 Sep 2025 11:33:20 +0200 Subject: [PATCH 06/10] Feature/more GitHub power (#44) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Split CLI code of all commands: ### Changed - **Major CLI restructuring**: Reorganized flat command structure into logical command groups for better organization and maintainability - `repox` → `repox convert` (repository processing commands) - `swe-*` commands → `swe *` subcommands (e.g., `swe-from-repo` → `swe from-repo`) - `validate` → `validation validate` (with additional `validation dry-run` and `validation check-config` options) - **Improved CLI architecture**: Extracted command implementations from main CLI module into co-located packages (`cocode/repox/repox_cli.py`, `cocode/swe/swe_cli.py`, etc.) for better code organization - **Updated documentation**: All examples and references updated to reflect new command structure ### Added - Command group structure with `app.add_typer()` for better CLI organization - `cocode/common.py` module with shared utilities (`PipeCode` enum, `validate_repo_path()`, `get_output_dir()`) - Alternative command names for flexibility (e.g., `repox repo` alongside `repox convert`) ### Deprecated - Direct `cocode validate` command (still works but shows deprecation notice; use `cocode validation validate` instead) **Migration**: Replace hyphens with spaces in SWE commands (e.g., `swe-from-repo` → `swe from-repo`) and use `repox convert` instead of `repox`. All old functionality remains available in the new structure. * use github repos with temp clones * Fix linting * Cleanup imports * Lint format --- CHANGELOG.md | 22 + CLAUDE.md | 1 + CLI_README.md | 159 ++++-- README.md | 20 +- cocode/cli.py | 473 +----------------- cocode/common.py | 104 ++++ cocode/github/github_repo_manager.py | 322 ++++++++++++ cocode/github/github_wrapper.py | 46 ++ cocode/repox/repox_cli.py | 126 +++++ cocode/swe/swe_cli.py | 350 +++++++++++++ cocode/validation_cli.py | 39 ++ docs/pages/commands.md | 30 +- tests/conftest.py | 29 +- tests/integration/test_github_integration.py | 122 +++++ tests/unit/github/test_github_repo_manager.py | 326 ++++++++++++ tests/unit/test_common.py | 69 +++ 16 files changed, 1711 insertions(+), 527 deletions(-) create mode 100644 cocode/common.py create mode 100644 cocode/github/github_repo_manager.py create mode 100644 cocode/repox/repox_cli.py create mode 100644 cocode/swe/swe_cli.py create mode 100644 cocode/validation_cli.py create mode 100644 tests/integration/test_github_integration.py create mode 100644 tests/unit/github/test_github_repo_manager.py create mode 100644 tests/unit/test_common.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 185692f..180344d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +Here's a more concise and better explained changelog: + +## Unreleased + +### Changed +- **Major CLI restructuring**: Reorganized flat command structure into logical command groups for better organization and maintainability + - `repox` → `repox convert` (repository processing commands) + - `swe-*` commands → `swe *` subcommands (e.g., `swe-from-repo` → `swe from-repo`) + - `validate` → `validation validate` (with additional `validation dry-run` and `validation check-config` options) +- **Improved CLI architecture**: Extracted command implementations from main CLI module into co-located packages (`cocode/repox/repox_cli.py`, `cocode/swe/swe_cli.py`, etc.) for better code organization +- **Updated documentation**: All examples and references updated to reflect new command structure + +### Added +- Command group structure with `app.add_typer()` for better CLI organization +- `cocode/common.py` module with shared utilities (`PipeCode` enum, `validate_repo_path()`, `get_output_dir()`) +- Alternative command names for flexibility (e.g., `repox repo` alongside `repox convert`) + +### Deprecated +- Direct `cocode validate` command (still works but shows deprecation notice; use `cocode validation validate` instead) + +**Migration**: Replace hyphens with spaces in SWE commands (e.g., `swe-from-repo` → `swe from-repo`) and use `repox convert` instead of `repox`. All old functionality remains available in the new structure. + ## [v0.1.3] - 2025-09-06 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 993a025..76c6eeb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -118,6 +118,7 @@ Always fix any issues reported by these tools before proceeding. - **Pipelines**: `cocode/pipelex_libraries/pipelines/` - **Tests**: `tests/` directory - **Documentation**: `docs/` directory + # Pipeline Guide - Always first write your "plan" in natural langage, then transcribe it in pipelex. diff --git a/CLI_README.md b/CLI_README.md index e923b3f..ace3a4f 100644 --- a/CLI_README.md +++ b/CLI_README.md @@ -8,68 +8,82 @@ Here are some common CoCode command examples to get you started: ```bash # Basic repository analysis -cocode repox --output-filename cocode_test.txt +cocode repox convert --output-filename cocode_test.txt # Analyze external project -cocode repox ../pipelex-cookbook/ --output-filename pipelex-cookbook.txt +cocode repox convert ../pipelex-cookbook/ --output-filename pipelex-cookbook.txt # Extract examples with specific Python rule -cocode repox ../pipelex-cookbook/ --output-filename "pipelex-cookbook-examples.txt" \ +cocode repox convert ../pipelex-cookbook/ --output-filename "pipelex-cookbook-examples.txt" \ --path-pattern "examples" --python-rule integral --include-pattern "*.py" # Extract Python imports from tools directory -cocode repox ../pipelex/ --output-filename "pipelex-tools-imports.txt" \ +cocode repox convert ../pipelex/ --output-filename "pipelex-tools-imports.txt" \ --path-pattern "tools" --python-rule imports --output-style import_list \ --include-pattern "*.py" # Analyze test interfaces -cocode repox ../pipelex/ --output-filename "pipelex-tests.txt" \ +cocode repox convert ../pipelex/ --output-filename "pipelex-tests.txt" \ --path-pattern "tests" --python-rule interface --include-pattern "*.py" # Extract cursor rules -cocode repox ../pipelex/ --output-filename "pipelex-cursor-rules.txt" \ +cocode repox convert ../pipelex/ --output-filename "pipelex-cursor-rules.txt" \ --path-pattern ".cursor/rules" --include-pattern "*.mdc" # Extract documentation -cocode repox ../pipelex/ --output-filename "pipelex-docs.txt" \ +cocode repox convert ../pipelex/ --output-filename "pipelex-docs.txt" \ --path-pattern "docs" --include-pattern "*.md" # Tree structure only -cocode repox ../pipelex/ --output-filename "pipelex-docs-tree.txt" \ +cocode repox convert ../pipelex/ --output-filename "pipelex-docs-tree.txt" \ --path-pattern "docs" --include-pattern "*.md" --output-style tree # Flat documentation with exclusions -cocode repox ../pipelex/ --output-filename "pipelex-docs.txt" \ +cocode repox convert ../pipelex/ --output-filename "pipelex-docs.txt" \ --path-pattern "docs" --include-pattern "*.md" \ --ignore-pattern "contributing.md" --ignore-pattern "CODE_OF_CONDUCT.md" \ --ignore-pattern "changelog.md" --ignore-pattern "license.md" \ --output-style flat -# SWE analysis: Extract fundamentals -cocode swe-from-repo extract_fundamentals ../pipelex/ \ +# SWE analysis: Extract fundamentals from local repo +cocode swe from-repo extract_fundamentals ../pipelex/ \ --path-pattern "docs" --include-pattern "*.md" \ --output-filename "fundamentals.json" -# SWE analysis: Extract onboarding documentation -cocode swe-from-repo extract_onboarding_documentation ../pipelex/ \ +# SWE analysis: Extract fundamentals from GitHub repo +cocode swe from-repo extract_fundamentals requests/requests \ + --path-pattern "docs" --include-pattern "*.md" \ + --output-filename "requests-fundamentals.json" + +# SWE analysis: Extract onboarding documentation from local repo +cocode swe from-repo extract_onboarding_documentation ../pipelex/ \ --path-pattern "docs" --include-pattern "*.md" \ --output-filename "docs-structured.json" +# SWE analysis: Extract onboarding documentation from GitHub repo with full URL +cocode swe from-repo extract_onboarding_documentation https://github.com/psf/black \ + --path-pattern "docs" --include-pattern "*.md" \ + --output-filename "black-docs-structured.json" + # SWE analysis: Comprehensive documentation extraction -cocode swe-from-repo extract_onboarding_documentation ../pipelex/ \ +cocode swe from-repo extract_onboarding_documentation ../pipelex/ \ --include-pattern "*.md" --include-pattern "*.mdc" \ --include-pattern "Makefile" --include-pattern "mkdocs.yml" \ --include-pattern "pyproject.toml" --include-pattern ".env.example" \ --output-filename "docs-structured.json" # SWE analysis: Extract features recap from file -cocode swe-from-file extract_features_recap ./results/pipelex-docs.txt \ +cocode swe from-file extract_features_recap ./results/pipelex-docs.txt \ --output-filename "pipelex-features-recap.md" -# SWE analysis: Generate changelog from git diff -cocode swe-from-repo-diff write_changelog v0.2.4 ../pipelex-cookbook/ \ +# SWE analysis: Generate changelog from git diff (local repo) +cocode swe from-repo-diff write_changelog v0.2.4 ../pipelex-cookbook/ \ --output-filename "changelog.md" +# SWE analysis: Generate changelog from GitHub repo +cocode swe from-repo-diff write_changelog v2.0.0 requests/requests \ + --output-filename "requests-changelog.md" + # GitHub operations cocode github auth # Check authentication status cocode github repo-info pipelex/cocode # Get repository information @@ -82,6 +96,34 @@ cocode github sync-labels pipelex/cocode ./labels.json # Sync labels CoCode provides powerful tools for repository analysis and Software Engineering automation through a command-line interface. It can convert repository structures to text files and perform AI-powered analysis using configurable pipelines. +### GitHub Repository Support + +CoCode supports analyzing both local repositories and GitHub repositories directly. You can specify GitHub repositories in several formats: + +- **Short format**: `owner/repo` (e.g., `microsoft/vscode`) +- **Full HTTPS URL**: `https://github.com/owner/repo` +- **SSH URL**: `git@github.com:owner/repo.git` +- **Branch-specific**: `owner/repo@branch` or full URLs with `/tree/branch` + +**Features:** +- **Smart Caching**: Repositories are cached locally for faster subsequent analysis +- **Authentication**: Supports GitHub Personal Access Tokens (PAT) and GitHub CLI authentication +- **Private Repositories**: Access private repositories with proper authentication +- **Shallow Cloning**: Fast cloning with minimal history for analysis purposes +- **Branch Support**: Analyze specific branches or tags + +**Authentication Setup:** +```bash +# Option 1: Set environment variable +export GITHUB_PAT=your_personal_access_token + +# Option 2: Use GitHub CLI (recommended) +gh auth login + +# Verify authentication +cocode github auth +``` + ## Installation & Setup ```bash @@ -98,22 +140,22 @@ Convert repository structure and contents to text files for analysis. **Basic Usage:** ```bash # Analyze current directory -cocode repox +cocode repox convert # Specify output file -cocode repox --output-filename my-repo.txt +cocode repox convert --output-filename my-repo.txt # Analyze external repository -cocode repox ../my-project/ --output-filename project-analysis.txt +cocode repox convert ../my-project/ --output-filename project-analysis.txt ``` **Advanced Filtering:** ```bash # Filter by file patterns -cocode repox --include-pattern "*.py" --python-rule interface +cocode repox convert --include-pattern "*.py" --python-rule interface # Extract Python imports from specific directory -cocode repox ../pipelex/ \ +cocode repox convert ../pipelex/ \ --output-filename "pipelex-tools-imports.txt" \ --path-pattern "tools" \ --python-rule imports \ @@ -121,7 +163,7 @@ cocode repox ../pipelex/ \ --include-pattern "*.py" # Analyze documentation with filtering -cocode repox ../project/ \ +cocode repox convert ../project/ \ --output-filename "docs.txt" \ --path-pattern "docs" \ --include-pattern "*.md" \ @@ -139,19 +181,20 @@ cocode repox ../project/ \ - `--python-rule, -p`: Python processing rule - `--output-style, -s`: Output format -### `swe-from-repo` - SWE Analysis from Repository +### `swe from-repo` - SWE Analysis from Repository -Perform Software Engineering analysis on repositories using AI pipelines. +Perform Software Engineering analysis on repositories using AI pipelines. Supports both local repositories and GitHub repositories. +**Local Repository Examples:** ```bash # Extract fundamentals from documentation -cocode swe-from-repo extract_fundamentals ../pipelex/ \ +cocode swe from-repo extract_fundamentals ../pipelex/ \ --path-pattern "docs" \ --include-pattern "*.md" \ --output-filename "fundamentals.json" # Extract comprehensive documentation structure -cocode swe-from-repo extract_onboarding_documentation ../pipelex/ \ +cocode swe from-repo extract_onboarding_documentation ../pipelex/ \ --include-pattern "*.md" \ --include-pattern "*.mdc" \ --include-pattern "Makefile" \ @@ -161,69 +204,89 @@ cocode swe-from-repo extract_onboarding_documentation ../pipelex/ \ --output-filename "docs-structured.json" # Dry run to test pipeline -cocode swe-from-repo extract_fundamentals . --dry +cocode swe from-repo extract_fundamentals . --dry ``` -### `swe-from-file` - SWE Analysis from File +**GitHub Repository Examples:** +```bash +# Analyze public GitHub repository (short format) +cocode swe from-repo extract_fundamentals requests/requests \ + --output-filename "requests-fundamentals.txt" + +# Analyze with full GitHub URL +cocode swe from-repo extract_onboarding_documentation https://github.com/psf/black \ + --output-filename "black-onboarding.txt" + +# Analyze specific branch +cocode swe from-repo extract_coding_standards pallets/click@main \ + --output-filename "click-standards.txt" + +# Focus on documentation from GitHub repo +cocode swe from-repo extract_fundamentals pytest-dev/pytest \ + --path-pattern "doc" --include-pattern "*.rst" --include-pattern "*.md" \ + --output-filename "pytest-docs-analysis.txt" +``` + +### `swe from-file` - SWE Analysis from File Process SWE analysis from existing text files. ```bash # Extract features recap from documentation -cocode swe-from-file extract_features_recap ./results/docs.txt \ +cocode swe from-file extract_features_recap ./results/docs.txt \ --output-filename "features-recap.md" ``` -### `swe-from-repo-diff` - SWE Analysis from Git Diff +### `swe from-repo-diff` - SWE Analysis from Git Diff Analyze git diffs using AI pipelines. ```bash # Generate changelog from git diff -cocode swe-from-repo-diff write_changelog v0.2.4 ../project/ \ +cocode swe from-repo-diff write_changelog v0.2.4 ../project/ \ --output-filename "changelog.md" # With ignore patterns -cocode swe-from-repo-diff write_changelog v1.0.0 . \ +cocode swe from-repo-diff write_changelog v1.0.0 . \ --ignore-patterns "*.log" \ --ignore-patterns "temp/" \ --output-filename "CHANGELOG.md" ``` -### `swe-doc-update` - Documentation Update Suggestions +### `swe doc-update` - Documentation Update Suggestions This command generates documentation update suggestions based on the differences detected in the git repository. **Usage**: ```bash -cocode swe-doc-update +cocode swe doc-update ``` **Examples**: ```bash -cocode swe-doc-update --help +cocode swe doc-update --help ``` -### `swe-doc-proofread` - Documentation Proofreading +### `swe doc-proofread` - Documentation Proofreading Systematically proofread documentation against actual codebase to find inconsistencies that could break user code or cause major confusion. **Usage**: ```bash # Proofread docs directory against current repository -cocode swe-doc-proofread +cocode swe doc-proofread # Specify custom documentation directory -cocode swe-doc-proofread --doc-dir documentation +cocode swe doc-proofread --doc-dir documentation # Proofread external project documentation -cocode swe-doc-proofread ../my-project/ --doc-dir docs --output-filename my-project-issues +cocode swe doc-proofread ../my-project/ --doc-dir docs --output-filename my-project-issues # Focus on specific file patterns in codebase analysis -cocode swe-doc-proofread --include-pattern "*.py" --include-pattern "*.ts" +cocode swe doc-proofread --include-pattern "*.py" --include-pattern "*.ts" # Exclude certain patterns from codebase analysis -cocode swe-doc-proofread --ignore-pattern "test_*" --ignore-pattern "*.md" +cocode swe doc-proofread --ignore-pattern "test_*" --ignore-pattern "*.md" ``` **Options:** @@ -327,11 +390,21 @@ cocode github sync-labels pipelex/cocode ./labels.json --delete-extra - `--delete-extra`: Remove labels not in the JSON file - `--limit`: Maximum number of items to display (for list commands) -### `validate` - Configuration Validation +### `validation` - Configuration Validation Validate setup and pipelines. ```bash +# Validate configuration and pipelines +cocode validation validate + +# Run dry validation without full setup +cocode validation dry-run + +# Check configuration only +cocode validation check-config + +# Backward compatibility (deprecated) cocode validate ``` diff --git a/README.md b/README.md index 900b3ea..06f07b5 100644 --- a/README.md +++ b/README.md @@ -40,16 +40,16 @@ Some complex pipelines require GCP credentials (See [GCP credentials](https://do ### Automatic Documentation & Release Features ```bash # Update documentation based on code changes -cocode swe-doc-update v1.0.0 path/to/your/local/repository +cocode swe doc-update v1.0.0 path/to/your/local/repository # Proofread documentation against codebase -cocode swe-doc-proofread --doc-dir docs path/to/your/local/repository # Requires GCP credentials for Gemini +cocode swe doc-proofread --doc-dir docs path/to/your/local/repository # Requires GCP credentials for Gemini # Generate changelog from version diff -cocode swe-from-repo-diff write_changelog v1.0.0 path/to/your/local/repository # Requires Anthropic API key for claude +cocode swe from-repo-diff write_changelog v1.0.0 path/to/your/local/repository # Requires Anthropic API key for claude # Update AI instructions (AGENTS.md, CLAUDE.md, cursor rules) based on code changes -cocode swe-ai-instruction-update v1.0.0 path/to/your/local/repository +cocode swe ai-instruction-update v1.0.0 path/to/your/local/repository # GitHub operations cocode github auth # Check GitHub authentication status @@ -133,28 +133,28 @@ If you run into issues or have suggestions, please check our [GitHub Issues](htt #### Basic Repository Analysis ```bash # Converts repositories into AI-readable text formats -cocode repox +cocode repox convert # Analyze specific project -cocode repox path/to/project --output-filename project-analysis.txt +cocode repox convert path/to/project --output-filename project-analysis.txt ``` #### Smart Code Extraction ```bash # Extract Python interfaces only -cocode repox --python-rule interface +cocode repox convert --python-rule interface # Analyze import dependencies -cocode repox --python-rule imports --output-style import_list +cocode repox convert --python-rule imports --output-style import_list ``` #### AI-Powered Analysis ```bash # Extract project fundamentals -cocode swe-from-repo extract_fundamentals . --output-filename overview.json +cocode swe from-repo extract_fundamentals . --output-filename overview.json # Generate feature documentation -cocode swe-from-file extract_features_recap ./analysis.txt --output-filename features.md +cocode swe from-file extract_features_recap ./analysis.txt --output-filename features.md ``` ## 🔧 Configuration diff --git a/cocode/cli.py b/cocode/cli.py index 90a0333..001f77d 100644 --- a/cocode/cli.py +++ b/cocode/cli.py @@ -2,88 +2,19 @@ CLI interface for cocode. """ -import asyncio -from pathlib import Path -from typing import Annotated, List, Optional +from typing import Optional import typer from click import Command, Context -from pipelex import log -from pipelex.core.pipes.pipe_run_params import PipeRunMode -from pipelex.hub import get_pipeline_tracker -from pipelex.pipe_works.pipe_dry import dry_run_all_pipes from pipelex.pipelex import Pipelex -from pipelex.tools.misc.file_utils import path_exists -from pipelex.types import StrEnum from typer import Context as TyperContext from typer.core import TyperGroup from typing_extensions import override from cocode.github.github_cli import github_app -from cocode.repox.models import OutputStyle -from cocode.repox.process_python import PythonProcessingRule -from cocode.repox.repox_cmd import repox_command -from cocode.repox.repox_processor import RESULTS_DIR -from cocode.swe.swe_cmd import ( - swe_ai_instruction_update_from_diff, - swe_doc_proofread, - swe_doc_update_from_diff, - swe_from_file, - swe_from_repo, - swe_from_repo_diff, -) - - -class PipeCode(StrEnum): - EXTRACT_ONBOARDING_DOCUMENTATION = "extract_onboarding_documentation" - EXTRACT_FUNDAMENTALS = "extract_fundamentals" - EXTRACT_ENVIRONMENT_BUILD = "extract_environment_build" - EXTRACT_CODING_STANDARDS = "extract_coding_standards" - EXTRACT_TEST_STRATEGY = "extract_test_strategy" - EXTRACT_CONTEXTUAL_GUIDELINES = "extract_contextual_guidelines" - EXTRACT_COLLABORATION = "extract_collaboration" - EXTRACT_FEATURES_RECAP = "extract_features_recap" - - DOC_PROOFREAD = "doc_proofread" - DOC_UPDATE = "doc_update" - AI_INSTRUCTION_UPDATE = "ai_instruction_update" - - # SWE diff analysis - WRITE_CHANGELOG = "write_changelog" - WRITE_CHANGELOG_ENHANCED = "write_changelog_enhanced" - - # Text utilities - GENERATE_SPLIT_IDENTIFIERS = "generate_split_identifiers" - - # SWE docs consistency check - CHECK_DOCS_INCONSISTENCIES = "check_doc_inconsistencies" - - -def _get_pipe_descriptions() -> str: - """Generate help text with pipe descriptions from TOML.""" - descriptions = { - "extract_onboarding_documentation": "Extract comprehensive onboarding documentation from software project docs", - "extract_fundamentals": "Extract fundamental project information from documentation", - "extract_environment_build": "Extract environment setup and build information from documentation", - "extract_coding_standards": "Extract code quality and style information from documentation", - "extract_test_strategy": "Extract testing strategy and procedures from documentation", - "extract_contextual_guidelines": "Extract contextual development guidelines from documentation", - "extract_collaboration": "Extract collaboration and workflow information from documentation", - "extract_features_recap": "Extract and analyze software features from documentation", - "doc_proofread": "Systematically proofread documentation against actual codebase to find inconsistencies", - "doc_update": "Generate documentation update suggestions for docs/ directory", - "ai_instruction_update": "Generate AI instruction update suggestions for AGENTS.md, CLAUDE.md, cursor rules", - "write_changelog": "Write a comprehensive changelog for a software project from git diff", - "write_changelog_enhanced": "Write a comprehensive changelog with draft and polish steps from git diff", - "generate_split_identifiers": "Analyze large text and generate optimal split identifiers", - "check_doc_inconsistencies": "Identify inconsistencies in a set of software engineering documents", - } - - help_text = "\n\n" - for code, description in descriptions.items(): - help_text += f" • [bold cyan]{code}[/bold cyan]: {description}\n\n\n" - - return help_text +from cocode.repox.repox_cli import repox_app +from cocode.swe.swe_cli import swe_app +from cocode.validation_cli import validation_app class CocodeCLI(TyperGroup): @@ -114,7 +45,10 @@ def get_command(self, ctx: Context, cmd_name: str) -> Optional[Command]: cls=CocodeCLI, ) -# Add GitHub command group +# Add command groups +app.add_typer(repox_app, name="repox", help="Repository processing and analysis commands") +app.add_typer(swe_app, name="swe", help="Software Engineering analysis and automation commands") +app.add_typer(validation_app, name="validation", help="Pipeline validation and setup commands") app.add_typer(github_app, name="github", help="GitHub-related operations and utilities") @@ -127,396 +61,13 @@ def main(ctx: TyperContext) -> None: print(ctx.get_help()) +# Keep the original validate command for backward compatibility @app.command() def validate() -> None: - """Run the setup sequence.""" - Pipelex.get_instance().validate_libraries() - asyncio.run(dry_run_all_pipes()) - log.info("Setup sequence passed OK, config and pipelines are validated.") - - -def _validate_repo_path(repo_path: str) -> str: - """Validate and convert repo_path to absolute path.""" - repo_path = str(Path(repo_path).resolve()) - - if not path_exists(repo_path): - log.error(f"[ERROR] Repo path '{repo_path}' does not exist") - raise typer.Exit(code=1) - - return repo_path - - -def _get_output_dir(output_dir: Optional[str]) -> str: - """Get output directory from parameter or config.""" - if output_dir is None: - return RESULTS_DIR - return output_dir - - -@app.command() -def repox( - repo_path: Annotated[ - str, - typer.Argument(help="Input directory path", exists=True, file_okay=False, dir_okay=True, resolve_path=True), - ] = ".", - output_dir: Annotated[ - Optional[str], - typer.Option("--output-dir", "-o", help="Output directory path. Use 'stdout' to print to console. Defaults to config value if not provided"), - ] = None, - output_filename: Annotated[ - str, - typer.Option("--output-filename", "-n", help="Output filename"), - ] = "repo-to-text.txt", - ignore_patterns: Annotated[ - Optional[List[str]], - typer.Option("--ignore-pattern", "-i", help="List of patterns to ignore (in gitignore format)"), - ] = None, - python_processing_rule: Annotated[ - PythonProcessingRule, - typer.Option("--python-rule", "-p", help="Python processing rule to apply", case_sensitive=False), - ] = PythonProcessingRule.INTERFACE, - output_style: Annotated[ - OutputStyle, - typer.Option( - "--output-style", "-s", help="One of: repo_map, flat (contents only), or import_list (for --python-rule imports)", case_sensitive=False - ), - ] = OutputStyle.REPO_MAP, - include_patterns: Annotated[ - Optional[List[str]], - typer.Option("--include-pattern", "-r", help="Optional pattern to filter files in the tree structure (glob pattern) - can be repeated"), - ] = None, - path_pattern: Annotated[ - Optional[str], - typer.Option("--path-pattern", "-pp", help="Optional pattern to filter paths in the tree structure (regex pattern)"), - ] = None, -) -> None: - """Convert repository structure and contents to a text file.""" - repo_path = _validate_repo_path(repo_path) - output_dir = _get_output_dir(output_dir) - to_stdout = output_dir == "stdout" - - repox_command( - repo_path=repo_path, - ignore_patterns=ignore_patterns, - include_patterns=include_patterns, - path_pattern=path_pattern, - python_processing_rule=python_processing_rule, - output_style=output_style, - output_filename=output_filename, - output_dir=output_dir, - to_stdout=to_stdout, - ) - - -@app.command("swe-from-repo") -def swe_from_repo_cmd( - pipe_code: Annotated[ - PipeCode, - typer.Argument(help=f"Pipeline code to execute for SWE analysis.\n\n{_get_pipe_descriptions()}"), - ] = PipeCode.EXTRACT_ONBOARDING_DOCUMENTATION, - repo_path: Annotated[ - str, - typer.Argument(help="Input directory path", exists=True, file_okay=False, dir_okay=True, resolve_path=True), - ] = ".", - output_dir: Annotated[ - Optional[str], - typer.Option("--output-dir", "-o", help="Output directory path. Use 'stdout' to print to console. Defaults to config value if not provided"), - ] = None, - output_filename: Annotated[ - str, - typer.Option("--output-filename", "-n", help="Output filename"), - ] = "swe-analysis.txt", - ignore_patterns: Annotated[ - Optional[List[str]], - typer.Option("--ignore-pattern", "-i", help="List of patterns to ignore (in gitignore format)"), - ] = None, - python_processing_rule: Annotated[ - PythonProcessingRule, - typer.Option("--python-rule", "-p", help="Python processing rule to apply", case_sensitive=False), - ] = PythonProcessingRule.INTERFACE, - output_style: Annotated[ - OutputStyle, - typer.Option( - "--output-style", "-s", help="One of: repo_map, flat (contents only), or import_list (for --python-rule imports)", case_sensitive=False - ), - ] = OutputStyle.REPO_MAP, - include_patterns: Annotated[ - Optional[List[str]], - typer.Option("--include-pattern", "-r", help="Optional pattern to filter files in the tree structure (glob pattern) - can be repeated"), - ] = None, - path_pattern: Annotated[ - Optional[str], - typer.Option("--path-pattern", "-pp", help="Optional pattern to filter paths in the tree structure (regex pattern)"), - ] = None, - dry_run: Annotated[ - bool, - typer.Option("--dry", help="Run pipeline in dry mode (no actual execution)"), - ] = False, -) -> None: - """Convert repository structure and contents to a text file with SWE analysis.""" - repo_path = _validate_repo_path(repo_path) - output_dir = _get_output_dir(output_dir) - to_stdout = output_dir == "stdout" - pipe_run_mode = PipeRunMode.DRY if dry_run else PipeRunMode.LIVE - - asyncio.run( - swe_from_repo( - pipe_code=pipe_code, - repo_path=repo_path, - ignore_patterns=ignore_patterns, - include_patterns=include_patterns, - path_pattern=path_pattern, - python_processing_rule=python_processing_rule, - output_style=output_style, - output_filename=output_filename, - output_dir=output_dir, - to_stdout=to_stdout, - pipe_run_mode=pipe_run_mode, - ) - ) - - -@app.command("swe-from-file") -def swe_from_file_cmd( - pipe_code: Annotated[ - PipeCode, - typer.Argument(help=f"Pipeline code to execute for SWE analysis.\n\n{_get_pipe_descriptions()}"), - ], - input_file_path: Annotated[ - str, - typer.Argument(help="Input text file path", exists=True, file_okay=True, dir_okay=False, resolve_path=True), - ], - output_dir: Annotated[ - Optional[str], - typer.Option("--output-dir", "-o", help="Output directory path. Use 'stdout' to print to console. Defaults to config value if not provided"), - ] = None, - output_filename: Annotated[ - str, - typer.Option("--output-filename", "-n", help="Output filename"), - ] = "swe-analysis.txt", - dry_run: Annotated[ - bool, - typer.Option("--dry", help="Run pipeline in dry mode (no actual execution)"), - ] = False, -) -> None: - """Process SWE analysis from an existing text file.""" - output_dir = _get_output_dir(output_dir) - to_stdout = output_dir == "stdout" - pipe_run_mode = PipeRunMode.DRY if dry_run else PipeRunMode.LIVE - - asyncio.run( - swe_from_file( - pipe_code=pipe_code, - input_file_path=input_file_path, - output_filename=output_filename, - output_dir=output_dir, - to_stdout=to_stdout, - pipe_run_mode=pipe_run_mode, - ) - ) - - -@app.command("swe-from-repo-diff") -def swe_from_repo_diff_cmd( - pipe_code: Annotated[ - str, - typer.Argument(help="Pipeline code to execute for SWE analysis"), - ], - version: Annotated[ - str, - typer.Argument(help="Git version/tag/commit to compare current version against"), - ], - repo_path: Annotated[ - str, - typer.Argument(help="Input directory path", exists=True, file_okay=False, dir_okay=True, resolve_path=True), - ] = ".", - output_dir: Annotated[ - Optional[str], - typer.Option("--output-dir", "-o", help="Output directory path. Use 'stdout' to print to console. Defaults to config value if not provided"), - ] = None, - output_filename: Annotated[ - str, - typer.Option("--output-filename", "-n", help="Output filename"), - ] = "swe-diff-analysis.md", - dry_run: Annotated[ - bool, - typer.Option("--dry", help="Run pipeline in dry mode (no actual execution)"), - ] = False, - ignore_patterns: Annotated[ - Optional[List[str]], - typer.Option( - "--ignore-pattern", "-i", help="Patterns to exclude from git diff (e.g., '*.log', 'temp/', 'build/'). Can be specified multiple times." - ), - ] = None, -) -> None: - """Process SWE analysis from git diff comparing current version to specified version.""" - repo_path = _validate_repo_path(repo_path) - output_dir = _get_output_dir(output_dir) - to_stdout = output_dir == "stdout" - pipe_run_mode = PipeRunMode.DRY if dry_run else PipeRunMode.LIVE - - asyncio.run( - swe_from_repo_diff( - pipe_code=pipe_code, - repo_path=repo_path, - version=version, - output_filename=output_filename, - output_dir=output_dir, - to_stdout=to_stdout, - pipe_run_mode=pipe_run_mode, - ignore_patterns=ignore_patterns, - ) - ) - get_pipeline_tracker().output_flowchart() - - -@app.command("swe-doc-update") -def swe_doc_update_cmd( - version: Annotated[ - str, - typer.Argument(help="Git version/tag/commit to compare current version against"), - ], - repo_path: Annotated[ - str, - typer.Argument(help="Input directory path", exists=True, file_okay=False, dir_okay=True, resolve_path=True), - ] = ".", - output_dir: Annotated[ - str, - typer.Option("--output-dir", "-o", help="Output directory path"), - ] = "results", - output_filename: Annotated[ - str, - typer.Option("--output-filename", "-n", help="Output filename"), - ] = "doc-update-suggestions.txt", - ignore_patterns: Annotated[ - Optional[List[str]], - typer.Option( - "--ignore-pattern", "-i", help="Patterns to exclude from git diff (e.g., '*.log', 'temp/', 'build/'). Can be specified multiple times." - ), - ] = None, - doc_dir: Annotated[ - Optional[str], - typer.Option("--doc-dir", "-d", help="Directory containing documentation files (e.g., 'docs', 'documentation')"), - ] = None, -) -> None: - """Generate documentation update suggestions for docs/ directory based on git diff analysis.""" - repo_path = _validate_repo_path(repo_path) - - asyncio.run( - swe_doc_update_from_diff( - repo_path=repo_path, - version=version, - output_filename=output_filename, - output_dir=output_dir, - ignore_patterns=ignore_patterns, - ) - ) - - get_pipeline_tracker().output_flowchart() - - -@app.command("swe-ai-instruction-update") -def swe_ai_instruction_update_cmd( - version: Annotated[ - str, - typer.Argument(help="Git version/tag/commit to compare current version against"), - ], - repo_path: Annotated[ - str, - typer.Argument(help="Input directory path", exists=True, file_okay=False, dir_okay=True, resolve_path=True), - ] = ".", - output_dir: Annotated[ - str, - typer.Option("--output-dir", "-o", help="Output directory path"), - ] = "results", - output_filename: Annotated[ - str, - typer.Option("--output-filename", "-n", help="Output filename"), - ] = "ai-instruction-update-suggestions.txt", - ignore_patterns: Annotated[ - Optional[List[str]], - typer.Option( - "--ignore-pattern", "-i", help="Patterns to exclude from git diff (e.g., '*.log', 'temp/', 'build/'). Can be specified multiple times." - ), - ] = None, -) -> None: - """Generate AI instruction update suggestions for AGENTS.md, CLAUDE.md, and cursor rules based on git diff analysis.""" - repo_path = _validate_repo_path(repo_path) - - asyncio.run( - swe_ai_instruction_update_from_diff( - repo_path=repo_path, - version=version, - output_filename=output_filename, - output_dir=output_dir, - ignore_patterns=ignore_patterns, - ) - ) - - get_pipeline_tracker().output_flowchart() - - -@app.command("swe-doc-proofread") -def swe_doc_proofread_cmd( - repo_path: Annotated[ - str, - typer.Argument(help="Input directory path", exists=True, file_okay=False, dir_okay=True, resolve_path=True), - ] = ".", - output_dir: Annotated[ - str, - typer.Option("--output-dir", "-o", help="Output directory path"), - ] = "results", - output_filename: Annotated[ - str, - typer.Option("--output-filename", "-n", help="Output filename"), - ] = "doc-proofread-report", - doc_dir: Annotated[ - str, - typer.Option("--doc-dir", "-d", help="Directory containing documentation files"), - ] = "docs", - include_patterns: Annotated[ - Optional[List[str]], - typer.Option("--include-pattern", "-r", help="Patterns to include in codebase analysis (glob pattern) - can be repeated"), - ] = None, - ignore_patterns: Annotated[ - Optional[List[str]], - typer.Option("--ignore-pattern", "-i", help="Patterns to ignore in codebase analysis (gitignore format) - can be repeated"), - ] = None, -) -> None: - """Systematically proofread documentation against actual codebase to find inconsistencies.""" - repo_path = _validate_repo_path(repo_path) - - # Set default include patterns to focus on documentation and code - if include_patterns is None: - include_patterns = ["*.md", "*.py", "*.toml", "*.yaml", "*.yml", "*.json", "*.sh", "*.js", "*.ts"] - - # Set default ignore patterns to exclude noise - if ignore_patterns is None: - ignore_patterns = [ - "__pycache__/", - "*.pyc", - ".git/", - ".venv/", - "node_modules/", - "*.log", - "build/", - "dist/", - ".pytest_cache/", - "*.egg-info/", - ] - - asyncio.run( - swe_doc_proofread( - repo_path=repo_path, - doc_dir=doc_dir, - output_filename=output_filename, - output_dir=output_dir, - include_patterns=include_patterns, - ignore_patterns=ignore_patterns, - ) - ) + """Run the setup sequence. (Deprecated: use 'cocode validation validate' instead)""" + from cocode.validation_cli import validate as validation_validate - get_pipeline_tracker().output_flowchart() + validation_validate() if __name__ == "__main__": diff --git a/cocode/common.py b/cocode/common.py new file mode 100644 index 0000000..b6934a9 --- /dev/null +++ b/cocode/common.py @@ -0,0 +1,104 @@ +""" +Common utilities and types shared across CLI modules. +""" + +from pathlib import Path +from typing import Optional + +import typer +from pipelex import log +from pipelex.tools.misc.file_utils import path_exists +from pipelex.types import StrEnum + +from cocode.github.github_repo_manager import GitHubRepoManager +from cocode.repox.repox_processor import RESULTS_DIR + + +class PipeCode(StrEnum): + """Pipeline codes for SWE analysis operations.""" + + EXTRACT_ONBOARDING_DOCUMENTATION = "extract_onboarding_documentation" + EXTRACT_FUNDAMENTALS = "extract_fundamentals" + EXTRACT_ENVIRONMENT_BUILD = "extract_environment_build" + EXTRACT_CODING_STANDARDS = "extract_coding_standards" + EXTRACT_TEST_STRATEGY = "extract_test_strategy" + EXTRACT_CONTEXTUAL_GUIDELINES = "extract_contextual_guidelines" + EXTRACT_COLLABORATION = "extract_collaboration" + EXTRACT_FEATURES_RECAP = "extract_features_recap" + + DOC_PROOFREAD = "doc_proofread" + DOC_UPDATE = "doc_update" + AI_INSTRUCTION_UPDATE = "ai_instruction_update" + + # SWE diff analysis + WRITE_CHANGELOG = "write_changelog" + WRITE_CHANGELOG_ENHANCED = "write_changelog_enhanced" + + # Text utilities + GENERATE_SPLIT_IDENTIFIERS = "generate_split_identifiers" + + # SWE docs consistency check + CHECK_DOCS_INCONSISTENCIES = "check_doc_inconsistencies" + + +def get_pipe_descriptions() -> str: + """Generate help text with pipe descriptions from TOML.""" + descriptions = { + "extract_onboarding_documentation": "Extract comprehensive onboarding documentation from software project docs", + "extract_fundamentals": "Extract fundamental project information from documentation", + "extract_environment_build": "Extract environment setup and build information from documentation", + "extract_coding_standards": "Extract code quality and style information from documentation", + "extract_test_strategy": "Extract testing strategy and procedures from documentation", + "extract_contextual_guidelines": "Extract contextual development guidelines from documentation", + "extract_collaboration": "Extract collaboration and workflow information from documentation", + "extract_features_recap": "Extract and analyze software features from documentation", + "doc_proofread": "Systematically proofread documentation against actual codebase to find inconsistencies", + "doc_update": "Generate documentation update suggestions for docs/ directory", + "ai_instruction_update": "Generate AI instruction update suggestions for AGENTS.md, CLAUDE.md, cursor rules", + "write_changelog": "Write a comprehensive changelog for a software project from git diff", + "write_changelog_enhanced": "Write a comprehensive changelog with draft and polish steps from git diff", + "generate_split_identifiers": "Analyze large text and generate optimal split identifiers", + "check_doc_inconsistencies": "Identify inconsistencies in a set of software engineering documents", + } + + help_text = "\n\n" + for code, description in descriptions.items(): + help_text += f" • [bold cyan]{code}[/bold cyan]: {description}\n\n\n" + + return help_text + + +def validate_repo_path(repo_path: str) -> str: + """ + Validate and convert repo_path to absolute path. + + For GitHub URLs, this will clone the repository and return the local path. + For local paths, this validates the path exists. + """ + # Check if it's a GitHub URL or identifier + if GitHubRepoManager.is_github_url(repo_path): + log.info(f"Detected GitHub repository: {repo_path}") + repo_manager = GitHubRepoManager() + try: + local_path = repo_manager.get_local_repo_path(repo_path, shallow=True) + log.info(f"GitHub repository cloned to: {local_path}") + return local_path + except Exception as exc: + log.error(f"[ERROR] Failed to clone GitHub repository '{repo_path}': {exc}") + raise typer.Exit(code=1) from exc + + # Handle local path + repo_path = str(Path(repo_path).resolve()) + + if not path_exists(repo_path): + log.error(f"[ERROR] Repo path '{repo_path}' does not exist") + raise typer.Exit(code=1) + + return repo_path + + +def get_output_dir(output_dir: Optional[str]) -> str: + """Get output directory from parameter or config.""" + if output_dir is None: + return RESULTS_DIR + return output_dir diff --git a/cocode/github/github_repo_manager.py b/cocode/github/github_repo_manager.py new file mode 100644 index 0000000..6f5e43e --- /dev/null +++ b/cocode/github/github_repo_manager.py @@ -0,0 +1,322 @@ +""" +GitHub repository manager for cloning and caching repositories. +""" + +import re +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import List, Optional, Tuple +from urllib.parse import urlparse + +from pipelex import log +from pipelex.tools.environment import get_optional_env + +from .github_wrapper import GithubWrapper + + +class GitHubRepoManagerError(Exception): + """Exception raised for GitHub repository manager errors.""" + + pass + + +class GitHubRepoManager: + """Manages GitHub repository cloning and caching.""" + + def __init__(self, cache_dir: Optional[str] = None): + """ + Initialize the GitHub repository manager. + + Args: + cache_dir: Directory to cache cloned repositories. If None, uses system temp directory. + """ + if cache_dir is None: + # Use a subdirectory in the user's home directory for persistent caching + home_dir = Path.home() + cache_dir = str(home_dir / ".cocode" / "cache" / "repos") + + self.cache_dir = Path(cache_dir) + self.cache_dir.mkdir(parents=True, exist_ok=True) + self.github_wrapper = GithubWrapper() + + @staticmethod + def is_github_url(url_or_path: str) -> bool: + """ + Check if the given string is a GitHub URL or repository identifier. + + Args: + url_or_path: String that might be a GitHub URL or local path + + Returns: + True if it's a GitHub URL or repo identifier, False otherwise + """ + # Check for full GitHub URLs + if url_or_path.startswith(("https://github.com/", "http://github.com/", "git@github.com:")): + return True + + # Check for short format (owner/repo) + if "/" in url_or_path and not url_or_path.startswith((".", "/", "~")): + # Simple heuristic: contains slash but doesn't look like a file path + parts = url_or_path.split("/") + if len(parts) == 2 and all(part.strip() for part in parts): + # Additional check: shouldn't contain typical file extensions or multiple levels + if not any(part.endswith((".txt", ".md", ".py", ".js", ".json", ".xml", ".html", ".css")) for part in parts): + # Basic validation: both parts should be non-empty after stripping + return True + + return False + + @staticmethod + def parse_github_url(url_or_identifier: str) -> Tuple[str, str, Optional[str]]: + """ + Parse a GitHub URL or identifier to extract owner, repo, and branch. + + Args: + url_or_identifier: GitHub URL or owner/repo format + + Returns: + Tuple of (owner, repo, branch) where branch can be None + + Raises: + GitHubRepoManagerError: If the URL format is invalid + """ + branch = None + + # Handle different GitHub URL formats + if url_or_identifier.startswith("git@github.com:"): + # SSH format: git@github.com:owner/repo.git + match = re.match(r"git@github\.com:([^/]+)/([^/]+?)(?:\.git)?(?:/tree/(.+))?$", url_or_identifier) + if not match: + raise GitHubRepoManagerError(f"Invalid GitHub SSH URL format: {url_or_identifier}") + owner, repo, branch = match.groups() + + elif url_or_identifier.startswith(("https://github.com/", "http://github.com/")): + # HTTPS format: https://github.com/owner/repo[/tree/branch] + parsed = urlparse(url_or_identifier) + path_parts = parsed.path.strip("/").split("/") + + if len(path_parts) < 2: + raise GitHubRepoManagerError(f"Invalid GitHub URL format: {url_or_identifier}") + + owner, repo = path_parts[0], path_parts[1] + + # Remove .git extension if present + if repo.endswith(".git"): + repo = repo[:-4] + + # Check for branch specification in URL + if len(path_parts) >= 4 and path_parts[2] == "tree": + branch = "/".join(path_parts[3:]) + + elif "/" in url_or_identifier and not url_or_identifier.startswith((".", "/", "~")): + # Short format: owner/repo[@branch] + if "@" in url_or_identifier: + repo_part, branch = url_or_identifier.rsplit("@", 1) + else: + repo_part = url_or_identifier + + parts = repo_part.split("/") + if len(parts) != 2: + raise GitHubRepoManagerError(f"Invalid GitHub repository identifier format: {url_or_identifier}") + + owner, repo = parts + + else: + raise GitHubRepoManagerError(f"Invalid GitHub URL or identifier: {url_or_identifier}") + + return owner, repo, branch + + def _get_cache_path(self, owner: str, repo: str) -> Path: + """Get the cache path for a repository.""" + return self.cache_dir / f"{owner}_{repo}" + + def _get_clone_url(self, owner: str, repo: str) -> str: + """ + Get the appropriate clone URL based on available authentication. + + Args: + owner: Repository owner + repo: Repository name + + Returns: + Clone URL (HTTPS with token if available, otherwise HTTPS public) + """ + # Try to get GitHub token for private repos + token = get_optional_env("GITHUB_PAT") + + if not token: + try: + # Try GitHub CLI + result = subprocess.run(["gh", "auth", "token"], capture_output=True, text=True, check=True) + token = result.stdout.strip() + except (subprocess.CalledProcessError, FileNotFoundError): + # No authentication available, use public HTTPS + pass + + if token: + return f"https://{token}@github.com/{owner}/{repo}.git" + else: + return f"https://github.com/{owner}/{repo}.git" + + def _clone_repository(self, owner: str, repo: str, branch: Optional[str], cache_path: Path, shallow: bool = True) -> None: + """ + Clone a repository to the cache directory. + + Args: + owner: Repository owner + repo: Repository name + branch: Optional branch to clone + cache_path: Path where to clone the repository + shallow: Whether to perform a shallow clone + + Raises: + GitHubRepoManagerError: If cloning fails + """ + clone_url = self._get_clone_url(owner, repo) + + # Prepare git clone command + cmd = ["git", "clone"] + + if shallow: + cmd.append("--depth=1") + + if branch: + cmd.extend(["--branch", branch]) + + cmd.extend([clone_url, str(cache_path)]) + + log.info(f"Cloning {owner}/{repo} to {cache_path}") + + try: + subprocess.run(cmd, capture_output=True, text=True, check=True, cwd=self.cache_dir) + log.info(f"Successfully cloned {owner}/{repo}") + except subprocess.CalledProcessError as exc: + error_msg = f"Failed to clone {owner}/{repo}: {exc.stderr}" + log.error(error_msg) + # Clean up partial clone if it exists + if cache_path.exists(): + shutil.rmtree(cache_path) + raise GitHubRepoManagerError(error_msg) from exc + + def _update_repository(self, cache_path: Path, branch: Optional[str] = None) -> None: + """ + Update an existing cached repository. + + Args: + cache_path: Path to the cached repository + branch: Optional branch to checkout + + Raises: + GitHubRepoManagerError: If update fails + """ + log.info(f"Updating cached repository at {cache_path}") + + try: + # Fetch latest changes + subprocess.run(["git", "fetch", "origin"], cwd=cache_path, capture_output=True, text=True, check=True) + + # Checkout specific branch if requested + if branch: + subprocess.run(["git", "checkout", branch], cwd=cache_path, capture_output=True, text=True, check=True) + + # Pull latest changes + subprocess.run(["git", "pull"], cwd=cache_path, capture_output=True, text=True, check=True) + + log.info(f"Successfully updated repository at {cache_path}") + + except subprocess.CalledProcessError as exc: + error_msg = f"Failed to update repository at {cache_path}: {exc.stderr}" + log.error(error_msg) + raise GitHubRepoManagerError(error_msg) from exc + + def get_local_repo_path( + self, + github_url_or_identifier: str, + force_refresh: bool = False, + shallow: bool = True, + temp_dir: bool = False, + ) -> str: + """ + Get a local path to a GitHub repository, cloning if necessary. + + Args: + github_url_or_identifier: GitHub URL or owner/repo format + force_refresh: If True, force a fresh clone even if cached + shallow: Whether to perform shallow clone (faster, less history) + temp_dir: If True, clone to a temporary directory instead of cache + + Returns: + Local path to the repository + + Raises: + GitHubRepoManagerError: If repository cannot be accessed + """ + owner, repo, branch = self.parse_github_url(github_url_or_identifier) + + if temp_dir: + # Clone to a temporary directory + temp_path = Path(tempfile.mkdtemp(prefix=f"cocode_{owner}_{repo}_")) + self._clone_repository(owner, repo, branch, temp_path, shallow=shallow) + return str(temp_path) + + cache_path = self._get_cache_path(owner, repo) + + # Check if we need to clone or update + if force_refresh and cache_path.exists(): + log.info(f"Force refresh requested, removing cached repository at {cache_path}") + shutil.rmtree(cache_path) + + if not cache_path.exists(): + # Clone the repository + self._clone_repository(owner, repo, branch, cache_path, shallow=shallow) + else: + # Repository exists, update it + try: + self._update_repository(cache_path, branch) + except GitHubRepoManagerError: + # If update fails, try a fresh clone + log.warning(f"Update failed, attempting fresh clone of {owner}/{repo}") + shutil.rmtree(cache_path) + self._clone_repository(owner, repo, branch, cache_path, shallow=shallow) + + return str(cache_path) + + def cleanup_cache(self, max_age_days: int = 7) -> None: + """ + Clean up old cached repositories. + + Args: + max_age_days: Remove cached repos older than this many days + """ + import time + + current_time = time.time() + cutoff_time = current_time - (max_age_days * 24 * 60 * 60) + + log.info(f"Cleaning up cached repositories older than {max_age_days} days") + + for repo_dir in self.cache_dir.iterdir(): + if repo_dir.is_dir(): + repo_mtime = repo_dir.stat().st_mtime + if repo_mtime < cutoff_time: + log.info(f"Removing old cached repository: {repo_dir}") + shutil.rmtree(repo_dir) + + def list_cached_repos(self) -> List[str]: + """ + List all cached repositories. + + Returns: + List of cached repository identifiers + """ + cached_repos: List[str] = [] + for repo_dir in self.cache_dir.iterdir(): + if repo_dir.is_dir(): + # Convert cache directory name back to owner/repo format + dir_name = repo_dir.name + if "_" in dir_name: + cached_repos.append(dir_name.replace("_", "/", 1)) + + return cached_repos diff --git a/cocode/github/github_wrapper.py b/cocode/github/github_wrapper.py index 39fc82f..1b2f198 100644 --- a/cocode/github/github_wrapper.py +++ b/cocode/github/github_wrapper.py @@ -110,3 +110,49 @@ def sync_labels( existing_labels[label_name].delete() return created_labels, updated_labels, deleted_labels + + def verify_repository_access(self, owner: str, repo: str) -> bool: + """ + Verify that the repository exists and is accessible. + + Args: + owner: Repository owner + repo: Repository name + + Returns: + True if repository is accessible, False otherwise + + Raises: + GithubWrapperError: If GitHub client not connected + """ + if not self.github_client: + raise GithubWrapperError("GitHub client not connected. Call connect() first") + + try: + self.github_client.get_repo(f"{owner}/{repo}") + return True + except github.GithubException: + return False + + def get_default_branch(self, owner: str, repo: str) -> str: + """ + Get the default branch name for a repository. + + Args: + owner: Repository owner + repo: Repository name + + Returns: + Default branch name (e.g., 'main', 'master') + + Raises: + GithubWrapperError: If GitHub client not connected or repository not found + """ + if not self.github_client: + raise GithubWrapperError("GitHub client not connected. Call connect() first") + + try: + repo_obj = self.github_client.get_repo(f"{owner}/{repo}") + return repo_obj.default_branch + except github.GithubException as exc: + raise GithubWrapperError(f"Repository '{owner}/{repo}' not found") from exc diff --git a/cocode/repox/repox_cli.py b/cocode/repox/repox_cli.py new file mode 100644 index 0000000..f521f38 --- /dev/null +++ b/cocode/repox/repox_cli.py @@ -0,0 +1,126 @@ +""" +Repository processing CLI commands. +""" + +from typing import Annotated, List, Optional + +import typer + +from cocode.common import get_output_dir, validate_repo_path + +from .models import OutputStyle +from .process_python import PythonProcessingRule +from .repox_cmd import repox_command + +repox_app = typer.Typer( + name="repox", + help="Repository processing and analysis commands", + add_completion=False, + rich_markup_mode="rich", +) + + +@repox_app.command("convert") +def repox_convert( + repo_path: Annotated[ + str, + typer.Argument(help="Input directory path", exists=True, file_okay=False, dir_okay=True, resolve_path=True), + ] = ".", + output_dir: Annotated[ + Optional[str], + typer.Option("--output-dir", "-o", help="Output directory path. Use 'stdout' to print to console. Defaults to config value if not provided"), + ] = None, + output_filename: Annotated[ + str, + typer.Option("--output-filename", "-n", help="Output filename"), + ] = "repo-to-text.txt", + ignore_patterns: Annotated[ + Optional[List[str]], + typer.Option("--ignore-pattern", "-i", help="List of patterns to ignore (in gitignore format)"), + ] = None, + python_processing_rule: Annotated[ + PythonProcessingRule, + typer.Option("--python-rule", "-p", help="Python processing rule to apply", case_sensitive=False), + ] = PythonProcessingRule.INTERFACE, + output_style: Annotated[ + OutputStyle, + typer.Option( + "--output-style", "-s", help="One of: repo_map, flat (contents only), or import_list (for --python-rule imports)", case_sensitive=False + ), + ] = OutputStyle.REPO_MAP, + include_patterns: Annotated[ + Optional[List[str]], + typer.Option("--include-pattern", "-r", help="Optional pattern to filter files in the tree structure (glob pattern) - can be repeated"), + ] = None, + path_pattern: Annotated[ + Optional[str], + typer.Option("--path-pattern", "-pp", help="Optional pattern to filter paths in the tree structure (regex pattern)"), + ] = None, +) -> None: + """Convert repository structure and contents to a text file.""" + repo_path = validate_repo_path(repo_path) + output_dir = get_output_dir(output_dir) + to_stdout = output_dir == "stdout" + + repox_command( + repo_path=repo_path, + ignore_patterns=ignore_patterns, + include_patterns=include_patterns, + path_pattern=path_pattern, + python_processing_rule=python_processing_rule, + output_style=output_style, + output_filename=output_filename, + output_dir=output_dir, + to_stdout=to_stdout, + ) + + +# Keep the original command name for backward compatibility +@repox_app.command("repo") +def repox_repo( + repo_path: Annotated[ + str, + typer.Argument(help="Input directory path", exists=True, file_okay=False, dir_okay=True, resolve_path=True), + ] = ".", + output_dir: Annotated[ + Optional[str], + typer.Option("--output-dir", "-o", help="Output directory path. Use 'stdout' to print to console. Defaults to config value if not provided"), + ] = None, + output_filename: Annotated[ + str, + typer.Option("--output-filename", "-n", help="Output filename"), + ] = "repo-to-text.txt", + ignore_patterns: Annotated[ + Optional[List[str]], + typer.Option("--ignore-pattern", "-i", help="List of patterns to ignore (in gitignore format)"), + ] = None, + python_processing_rule: Annotated[ + PythonProcessingRule, + typer.Option("--python-rule", "-p", help="Python processing rule to apply", case_sensitive=False), + ] = PythonProcessingRule.INTERFACE, + output_style: Annotated[ + OutputStyle, + typer.Option( + "--output-style", "-s", help="One of: repo_map, flat (contents only), or import_list (for --python-rule imports)", case_sensitive=False + ), + ] = OutputStyle.REPO_MAP, + include_patterns: Annotated[ + Optional[List[str]], + typer.Option("--include-pattern", "-r", help="Optional pattern to filter files in the tree structure (glob pattern) - can be repeated"), + ] = None, + path_pattern: Annotated[ + Optional[str], + typer.Option("--path-pattern", "-pp", help="Optional pattern to filter paths in the tree structure (regex pattern)"), + ] = None, +) -> None: + """Convert repository structure and contents to a text file.""" + repox_convert( + repo_path=repo_path, + output_dir=output_dir, + output_filename=output_filename, + ignore_patterns=ignore_patterns, + python_processing_rule=python_processing_rule, + output_style=output_style, + include_patterns=include_patterns, + path_pattern=path_pattern, + ) diff --git a/cocode/swe/swe_cli.py b/cocode/swe/swe_cli.py new file mode 100644 index 0000000..833db5a --- /dev/null +++ b/cocode/swe/swe_cli.py @@ -0,0 +1,350 @@ +""" +Software Engineering analysis CLI commands. +""" + +import asyncio +from typing import Annotated, List, Optional + +import typer +from pipelex.core.pipes.pipe_run_params import PipeRunMode +from pipelex.hub import get_pipeline_tracker + +from cocode.common import PipeCode, get_output_dir, get_pipe_descriptions, validate_repo_path +from cocode.repox.models import OutputStyle +from cocode.repox.process_python import PythonProcessingRule + +from .swe_cmd import ( + swe_ai_instruction_update_from_diff, + swe_doc_proofread, + swe_doc_update_from_diff, + swe_from_file, + swe_from_repo, + swe_from_repo_diff, +) + +swe_app = typer.Typer( + name="swe", + help="Software Engineering analysis and automation commands", + add_completion=False, + rich_markup_mode="rich", +) + + +@swe_app.command("from-repo") +def swe_from_repo_cmd( + pipe_code: Annotated[ + PipeCode, + typer.Argument(help=f"Pipeline code to execute for SWE analysis.\n\n{get_pipe_descriptions()}"), + ] = PipeCode.EXTRACT_ONBOARDING_DOCUMENTATION, + repo_path: Annotated[ + str, + typer.Argument(help="Repository path (local directory) or GitHub URL/identifier (owner/repo or https://github.com/owner/repo)"), + ] = ".", + output_dir: Annotated[ + Optional[str], + typer.Option("--output-dir", "-o", help="Output directory path. Use 'stdout' to print to console. Defaults to config value if not provided"), + ] = None, + output_filename: Annotated[ + str, + typer.Option("--output-filename", "-n", help="Output filename"), + ] = "swe-analysis.txt", + ignore_patterns: Annotated[ + Optional[List[str]], + typer.Option("--ignore-pattern", "-i", help="List of patterns to ignore (in gitignore format)"), + ] = None, + python_processing_rule: Annotated[ + PythonProcessingRule, + typer.Option("--python-rule", "-p", help="Python processing rule to apply", case_sensitive=False), + ] = PythonProcessingRule.INTERFACE, + output_style: Annotated[ + OutputStyle, + typer.Option( + "--output-style", "-s", help="One of: repo_map, flat (contents only), or import_list (for --python-rule imports)", case_sensitive=False + ), + ] = OutputStyle.REPO_MAP, + include_patterns: Annotated[ + Optional[List[str]], + typer.Option("--include-pattern", "-r", help="Optional pattern to filter files in the tree structure (glob pattern) - can be repeated"), + ] = None, + path_pattern: Annotated[ + Optional[str], + typer.Option("--path-pattern", "-pp", help="Optional pattern to filter paths in the tree structure (regex pattern)"), + ] = None, + dry_run: Annotated[ + bool, + typer.Option("--dry", help="Run pipeline in dry mode (no actual execution)"), + ] = False, +) -> None: + """Convert repository structure and contents to a text file with SWE analysis. Supports both local repositories and GitHub repositories.""" + repo_path = validate_repo_path(repo_path) + output_dir = get_output_dir(output_dir) + to_stdout = output_dir == "stdout" + pipe_run_mode = PipeRunMode.DRY if dry_run else PipeRunMode.LIVE + + asyncio.run( + swe_from_repo( + pipe_code=pipe_code, + repo_path=repo_path, + ignore_patterns=ignore_patterns, + include_patterns=include_patterns, + path_pattern=path_pattern, + python_processing_rule=python_processing_rule, + output_style=output_style, + output_filename=output_filename, + output_dir=output_dir, + to_stdout=to_stdout, + pipe_run_mode=pipe_run_mode, + ) + ) + + +@swe_app.command("from-file") +def swe_from_file_cmd( + pipe_code: Annotated[ + PipeCode, + typer.Argument(help=f"Pipeline code to execute for SWE analysis.\n\n{get_pipe_descriptions()}"), + ], + input_file_path: Annotated[ + str, + typer.Argument(help="Input text file path", exists=True, file_okay=True, dir_okay=False, resolve_path=True), + ], + output_dir: Annotated[ + Optional[str], + typer.Option("--output-dir", "-o", help="Output directory path. Use 'stdout' to print to console. Defaults to config value if not provided"), + ] = None, + output_filename: Annotated[ + str, + typer.Option("--output-filename", "-n", help="Output filename"), + ] = "swe-analysis.txt", + dry_run: Annotated[ + bool, + typer.Option("--dry", help="Run pipeline in dry mode (no actual execution)"), + ] = False, +) -> None: + """Process SWE analysis from an existing text file.""" + output_dir = get_output_dir(output_dir) + to_stdout = output_dir == "stdout" + pipe_run_mode = PipeRunMode.DRY if dry_run else PipeRunMode.LIVE + + asyncio.run( + swe_from_file( + pipe_code=pipe_code, + input_file_path=input_file_path, + output_filename=output_filename, + output_dir=output_dir, + to_stdout=to_stdout, + pipe_run_mode=pipe_run_mode, + ) + ) + + +@swe_app.command("from-repo-diff") +def swe_from_repo_diff_cmd( + pipe_code: Annotated[ + str, + typer.Argument(help="Pipeline code to execute for SWE analysis"), + ], + version: Annotated[ + str, + typer.Argument(help="Git version/tag/commit to compare current version against"), + ], + repo_path: Annotated[ + str, + typer.Argument(help="Repository path (local directory) or GitHub URL/identifier (owner/repo or https://github.com/owner/repo)"), + ] = ".", + output_dir: Annotated[ + Optional[str], + typer.Option("--output-dir", "-o", help="Output directory path. Use 'stdout' to print to console. Defaults to config value if not provided"), + ] = None, + output_filename: Annotated[ + str, + typer.Option("--output-filename", "-n", help="Output filename"), + ] = "swe-diff-analysis.md", + dry_run: Annotated[ + bool, + typer.Option("--dry", help="Run pipeline in dry mode (no actual execution)"), + ] = False, + ignore_patterns: Annotated[ + Optional[List[str]], + typer.Option( + "--ignore-pattern", "-i", help="Patterns to exclude from git diff (e.g., '*.log', 'temp/', 'build/'). Can be specified multiple times." + ), + ] = None, +) -> None: + """Process SWE analysis from git diff comparing current version to specified version. Supports both local repositories and GitHub repositories.""" + repo_path = validate_repo_path(repo_path) + output_dir = get_output_dir(output_dir) + to_stdout = output_dir == "stdout" + pipe_run_mode = PipeRunMode.DRY if dry_run else PipeRunMode.LIVE + + asyncio.run( + swe_from_repo_diff( + pipe_code=pipe_code, + repo_path=repo_path, + version=version, + output_filename=output_filename, + output_dir=output_dir, + to_stdout=to_stdout, + pipe_run_mode=pipe_run_mode, + ignore_patterns=ignore_patterns, + ) + ) + get_pipeline_tracker().output_flowchart() + + +@swe_app.command("doc-update") +def swe_doc_update_cmd( + version: Annotated[ + str, + typer.Argument(help="Git version/tag/commit to compare current version against"), + ], + repo_path: Annotated[ + str, + typer.Argument(help="Repository path (local directory) or GitHub URL/identifier (owner/repo or https://github.com/owner/repo)"), + ] = ".", + output_dir: Annotated[ + str, + typer.Option("--output-dir", "-o", help="Output directory path"), + ] = "results", + output_filename: Annotated[ + str, + typer.Option("--output-filename", "-n", help="Output filename"), + ] = "doc-update-suggestions.txt", + ignore_patterns: Annotated[ + Optional[List[str]], + typer.Option( + "--ignore-pattern", "-i", help="Patterns to exclude from git diff (e.g., '*.log', 'temp/', 'build/'). Can be specified multiple times." + ), + ] = None, + doc_dir: Annotated[ + Optional[str], + typer.Option("--doc-dir", "-d", help="Directory containing documentation files (e.g., 'docs', 'documentation')"), + ] = None, +) -> None: + """ + Generate documentation update suggestions for docs/ directory based on git diff analysis. + Supports both local repositories and GitHub repositories. + """ + repo_path = validate_repo_path(repo_path) + + asyncio.run( + swe_doc_update_from_diff( + repo_path=repo_path, + version=version, + output_filename=output_filename, + output_dir=output_dir, + ignore_patterns=ignore_patterns, + ) + ) + + get_pipeline_tracker().output_flowchart() + + +@swe_app.command("ai-instruction-update") +def swe_ai_instruction_update_cmd( + version: Annotated[ + str, + typer.Argument(help="Git version/tag/commit to compare current version against"), + ], + repo_path: Annotated[ + str, + typer.Argument(help="Repository path (local directory) or GitHub URL/identifier (owner/repo or https://github.com/owner/repo)"), + ] = ".", + output_dir: Annotated[ + str, + typer.Option("--output-dir", "-o", help="Output directory path"), + ] = "results", + output_filename: Annotated[ + str, + typer.Option("--output-filename", "-n", help="Output filename"), + ] = "ai-instruction-update-suggestions.txt", + ignore_patterns: Annotated[ + Optional[List[str]], + typer.Option( + "--ignore-pattern", "-i", help="Patterns to exclude from git diff (e.g., '*.log', 'temp/', 'build/'). Can be specified multiple times." + ), + ] = None, +) -> None: + """ + Generate AI instruction update suggestions for AGENTS.md, CLAUDE.md, and cursor rules based on git diff analysis. + Supports both local repositories and GitHub repositories. + """ + repo_path = validate_repo_path(repo_path) + + asyncio.run( + swe_ai_instruction_update_from_diff( + repo_path=repo_path, + version=version, + output_filename=output_filename, + output_dir=output_dir, + ignore_patterns=ignore_patterns, + ) + ) + + get_pipeline_tracker().output_flowchart() + + +@swe_app.command("doc-proofread") +def swe_doc_proofread_cmd( + repo_path: Annotated[ + str, + typer.Argument(help="Repository path (local directory) or GitHub URL/identifier (owner/repo or https://github.com/owner/repo)"), + ] = ".", + output_dir: Annotated[ + str, + typer.Option("--output-dir", "-o", help="Output directory path"), + ] = "results", + output_filename: Annotated[ + str, + typer.Option("--output-filename", "-n", help="Output filename"), + ] = "doc-proofread-report", + doc_dir: Annotated[ + str, + typer.Option("--doc-dir", "-d", help="Directory containing documentation files"), + ] = "docs", + include_patterns: Annotated[ + Optional[List[str]], + typer.Option("--include-pattern", "-r", help="Patterns to include in codebase analysis (glob pattern) - can be repeated"), + ] = None, + ignore_patterns: Annotated[ + Optional[List[str]], + typer.Option("--ignore-pattern", "-i", help="Patterns to ignore in codebase analysis (gitignore format) - can be repeated"), + ] = None, +) -> None: + """ + Systematically proofread documentation against actual codebase to find inconsistencies. + Supports both local repositories and GitHub repositories. + """ + repo_path = validate_repo_path(repo_path) + + # Set default include patterns to focus on documentation and code + if include_patterns is None: + include_patterns = ["*.md", "*.py", "*.toml", "*.yaml", "*.yml", "*.json", "*.sh", "*.js", "*.ts"] + + # Set default ignore patterns to exclude noise + if ignore_patterns is None: + ignore_patterns = [ + "__pycache__/", + "*.pyc", + ".git/", + ".venv/", + "node_modules/", + "*.log", + "build/", + "dist/", + ".pytest_cache/", + "*.egg-info/", + ] + + asyncio.run( + swe_doc_proofread( + repo_path=repo_path, + doc_dir=doc_dir, + output_filename=output_filename, + output_dir=output_dir, + include_patterns=include_patterns, + ignore_patterns=ignore_patterns, + ) + ) + + get_pipeline_tracker().output_flowchart() diff --git a/cocode/validation_cli.py b/cocode/validation_cli.py new file mode 100644 index 0000000..9ecb2e9 --- /dev/null +++ b/cocode/validation_cli.py @@ -0,0 +1,39 @@ +""" +Pipeline validation CLI commands. +""" + +import asyncio + +import typer +from pipelex import log +from pipelex.pipe_works.pipe_dry import dry_run_all_pipes +from pipelex.pipelex import Pipelex + +validation_app = typer.Typer( + name="validation", + help="Pipeline validation and setup commands", + add_completion=False, + rich_markup_mode="rich", +) + + +@validation_app.command("validate") +def validate() -> None: + """Run the setup sequence and validate all pipelines.""" + Pipelex.get_instance().validate_libraries() + asyncio.run(dry_run_all_pipes()) + log.info("Setup sequence passed OK, config and pipelines are validated.") + + +@validation_app.command("dry-run") +def dry_run() -> None: + """Run dry validation of all pipelines without full setup.""" + asyncio.run(dry_run_all_pipes()) + log.info("Dry run completed successfully.") + + +@validation_app.command("check-config") +def check_config() -> None: + """Validate Pipelex configuration and libraries.""" + Pipelex.get_instance().validate_libraries() + log.info("Configuration validation passed OK.") diff --git a/docs/pages/commands.md b/docs/pages/commands.md index 99d435b..b278223 100644 --- a/docs/pages/commands.md +++ b/docs/pages/commands.md @@ -4,12 +4,20 @@ title: Commands # Commands +CoCode uses organized command groups for better structure: + +- **`repox`** - Repository processing commands +- **`swe`** - Software Engineering analysis commands +- **`validation`** - Pipeline validation commands +- **`github`** - GitHub repository management + ## repox Convert repository to text format. ```bash -cocode repox [OPTIONS] [REPO_PATH] +cocode repox convert [OPTIONS] [REPO_PATH] +cocode repox repo [OPTIONS] [REPO_PATH] # Alternative command name ``` **Options:** @@ -22,12 +30,12 @@ cocode repox [OPTIONS] [REPO_PATH] - `-p, --python-rule` - Python processing: `interface`, `imports`, `integral` - `-s, --output-style` - Output format: `repo_map`, `flat`, `tree`, `import_list` -## swe-from-repo +## swe from-repo Analyze repository with AI pipelines. ```bash -cocode swe-from-repo [OPTIONS] PIPE_CODE REPO_PATH +cocode swe from-repo [OPTIONS] PIPE_CODE REPO_PATH ``` **Pipelines:** @@ -46,12 +54,12 @@ cocode swe-from-repo [OPTIONS] PIPE_CODE REPO_PATH - `--dry` - Dry run without API calls - Plus all filtering options from `repox` -## swe-from-file +## swe from-file Process text file with AI pipelines. ```bash -cocode swe-from-file [OPTIONS] PIPE_CODE INPUT_FILE +cocode swe from-file [OPTIONS] PIPE_CODE INPUT_FILE ``` **Pipelines:** @@ -60,12 +68,12 @@ cocode swe-from-file [OPTIONS] PIPE_CODE INPUT_FILE - `extract_fundamentals` - Extract project info - `extract_onboarding_documentation` - Structure docs -## swe-from-repo-diff +## swe from-repo-diff Analyze git diffs with AI. ```bash -cocode swe-from-repo-diff [OPTIONS] PIPE_CODE GIT_REF REPO_PATH +cocode swe from-repo-diff [OPTIONS] PIPE_CODE GIT_REF REPO_PATH ``` **Main pipeline:** @@ -79,12 +87,12 @@ cocode swe-from-repo-diff [OPTIONS] PIPE_CODE GIT_REF REPO_PATH - Ranges: `v1.0.0..v2.0.0` - Relative: `HEAD~10` -## swe-doc-proofread +## swe doc-proofread Proofread documentation against codebase to detect inconsistencies. ```bash -cocode swe-doc-proofread [OPTIONS] REPO_PATH +cocode swe doc-proofread [OPTIONS] REPO_PATH ``` **Purpose:** @@ -176,12 +184,12 @@ cocode github sync-labels [OPTIONS] REPO LABELS_FILE ] ``` -## swe-doc-update +## swe doc-update Update documentation based on code changes. ```bash -cocode swe-doc-update [OPTIONS] GIT_REF REPO_PATH +cocode swe doc-update [OPTIONS] GIT_REF REPO_PATH ``` **Purpose:** diff --git a/tests/conftest.py b/tests/conftest.py index cc6b30c..04334a3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,8 @@ +import logging + import pipelex.config import pipelex.pipelex import pytest -from pipelex import pretty_print from pipelex.config import get_config from rich import print from rich.console import Console @@ -20,7 +21,6 @@ def reset_pipelex_config_fixture(): pipelex_instance = pipelex.pipelex.Pipelex.make(relative_config_folder_path="./cocode/pipelex_libraries", from_file=False) pipelex_instance.validate_libraries() config = get_config() - pretty_print(config, title="Test config") assert isinstance(config, pipelex.config.PipelexConfig) assert config.project_name == "cocode" except Exception as exc: @@ -39,3 +39,28 @@ def pretty(): yield # Code to run after each test print("\n") + + +@pytest.fixture +def suppress_error_logs(): + """ + Fixture to suppress error logs during tests that expect failures. + + This prevents confusing error messages in test output when testing + expected failure scenarios (e.g., invalid repositories, network errors). + + Usage: + def test_expected_failure(self, mocker, suppress_error_logs): + # Test code that expects errors without showing error logs + """ + # Store original log level + logger = logging.getLogger("cocode") + original_level = logger.level + + # Set to CRITICAL to suppress INFO and ERROR logs + logger.setLevel(logging.CRITICAL) + + yield + + # Restore original log level + logger.setLevel(original_level) diff --git a/tests/integration/test_github_integration.py b/tests/integration/test_github_integration.py new file mode 100644 index 0000000..eefc90d --- /dev/null +++ b/tests/integration/test_github_integration.py @@ -0,0 +1,122 @@ +""" +Integration tests for GitHub functionality. + +These tests require network access and may use GitHub API. +Mark as slow or skip in CI if needed. +""" + +import os +import tempfile +from pathlib import Path + +import pytest + +from cocode.github.github_repo_manager import GitHubRepoManager, GitHubRepoManagerError + + +@pytest.mark.integration +@pytest.mark.gha_disabled +class TestGitHubIntegration: + """Integration tests for GitHub repository manager.""" + + def test_clone_public_repository(self): + """Test cloning a small public repository.""" + with tempfile.TemporaryDirectory() as temp_dir: + manager = GitHubRepoManager(cache_dir=temp_dir) + + # Use a small, stable public Python repository for testing + repo_path = manager.get_local_repo_path("jazzband/pip-tools", shallow=True) + + # Verify the repository was cloned + assert os.path.exists(repo_path) + assert os.path.isdir(repo_path) + + # Verify it's a git repository + git_dir = Path(repo_path) / ".git" + assert git_dir.exists() + + # Verify some expected files exist + readme_path = Path(repo_path) / "README.md" + assert readme_path.exists() + + # Verify it's a Python project + setup_py = Path(repo_path) / "setup.py" + pyproject_toml = Path(repo_path) / "pyproject.toml" + assert setup_py.exists() or pyproject_toml.exists() + + def test_clone_nonexistent_repository(self, suppress_error_logs: None) -> None: + """Test cloning a non-existent repository.""" + with tempfile.TemporaryDirectory() as temp_dir: + manager = GitHubRepoManager(cache_dir=temp_dir) + + with pytest.raises(GitHubRepoManagerError): + manager.get_local_repo_path("nonexistent-user/nonexistent-repo-12345", shallow=True) + + def test_cache_functionality(self): + """Test that caching works correctly.""" + with tempfile.TemporaryDirectory() as temp_dir: + manager = GitHubRepoManager(cache_dir=temp_dir) + + # First clone + repo_path1 = manager.get_local_repo_path("jazzband/pip-tools", shallow=True) + + # Second access should use cache + repo_path2 = manager.get_local_repo_path("jazzband/pip-tools", shallow=True) + + # Paths should be the same + assert repo_path1 == repo_path2 + + # Verify cache directory contains the repo + cached_repos = manager.list_cached_repos() + assert "jazzband/pip-tools" in cached_repos + + def test_force_refresh(self): + """Test force refresh functionality.""" + with tempfile.TemporaryDirectory() as temp_dir: + manager = GitHubRepoManager(cache_dir=temp_dir) + + # First clone + repo_path1 = manager.get_local_repo_path("jazzband/pip-tools", shallow=True) + + # Create a marker file to detect refresh + marker_file = Path(repo_path1) / "test_marker" + marker_file.touch() + + # Force refresh should remove the marker + repo_path2 = manager.get_local_repo_path("jazzband/pip-tools", force_refresh=True) + + assert repo_path1 == repo_path2 + assert not marker_file.exists() # Should be gone after refresh + + def test_temp_directory_mode(self): + """Test temporary directory mode.""" + manager = GitHubRepoManager() + + repo_path = manager.get_local_repo_path("jazzband/pip-tools", temp_dir=True, shallow=True) + + # Verify it's in a temporary location (not in cache) + # Check for common temp directory patterns across different OS + temp_indicators = ["/tmp", "temp", "/var/folders", "\\temp\\", "AppData\\Local\\Temp"] + assert any(indicator in repo_path.lower() for indicator in temp_indicators) + + # Verify the repository exists and is valid + assert os.path.exists(repo_path) + git_dir = Path(repo_path) / ".git" + assert git_dir.exists() + + def test_branch_specification(self): + """Test cloning specific branches.""" + with tempfile.TemporaryDirectory() as temp_dir: + manager = GitHubRepoManager(cache_dir=temp_dir) + + # First clone without branch to get the default branch + default_repo_path = manager.get_local_repo_path("jazzband/pip-tools", shallow=True) + assert os.path.exists(default_repo_path) + assert os.path.isdir(default_repo_path) + + # Now clone main branch specifically (most modern Python repos use main) + repo_path = manager.get_local_repo_path("jazzband/pip-tools@main", shallow=True, force_refresh=True) + + # Verify the repository was cloned + assert os.path.exists(repo_path) + assert os.path.isdir(repo_path) diff --git a/tests/unit/github/test_github_repo_manager.py b/tests/unit/github/test_github_repo_manager.py new file mode 100644 index 0000000..2ab0ece --- /dev/null +++ b/tests/unit/github/test_github_repo_manager.py @@ -0,0 +1,326 @@ +""" +Unit tests for GitHub repository manager functionality. +""" + +import os +import tempfile +from pathlib import Path + +import pytest +from pytest_mock import MockerFixture + +from cocode.github.github_repo_manager import GitHubRepoManager, GitHubRepoManagerError + + +class TestGitHubRepoManager: + """Test cases for GitHub repository manager.""" + + def test_is_github_url_https_format(self): + """Test detection of HTTPS GitHub URLs.""" + assert GitHubRepoManager.is_github_url("https://github.com/owner/repo") + assert GitHubRepoManager.is_github_url("http://github.com/owner/repo") + assert GitHubRepoManager.is_github_url("https://github.com/owner/repo.git") + assert GitHubRepoManager.is_github_url("https://github.com/owner/repo/tree/main") + + def test_is_github_url_ssh_format(self): + """Test detection of SSH GitHub URLs.""" + assert GitHubRepoManager.is_github_url("git@github.com:owner/repo.git") + assert GitHubRepoManager.is_github_url("git@github.com:owner/repo") + + def test_is_github_url_short_format(self): + """Test detection of short GitHub identifiers.""" + assert GitHubRepoManager.is_github_url("owner/repo") + assert GitHubRepoManager.is_github_url("microsoft/vscode") + assert GitHubRepoManager.is_github_url("facebook/react") + + # Test ambiguous cases that should be treated as GitHub repos + assert GitHubRepoManager.is_github_url("username/project") + assert GitHubRepoManager.is_github_url("org/library") + + def test_is_github_url_local_paths(self): + """Test that local paths are not detected as GitHub URLs.""" + assert not GitHubRepoManager.is_github_url("./local/path") + assert not GitHubRepoManager.is_github_url("/absolute/path") + assert not GitHubRepoManager.is_github_url("~/home/path") + assert not GitHubRepoManager.is_github_url("relative/path.txt") + assert not GitHubRepoManager.is_github_url("docs/readme.md") + assert not GitHubRepoManager.is_github_url(".") + assert not GitHubRepoManager.is_github_url("..") + + def test_is_github_url_invalid_formats(self): + """Test that invalid formats are not detected as GitHub URLs.""" + assert not GitHubRepoManager.is_github_url("just-a-string") + assert not GitHubRepoManager.is_github_url("owner/") + assert not GitHubRepoManager.is_github_url("/repo") + assert not GitHubRepoManager.is_github_url("") + + def test_parse_github_url_https_format(self): + """Test parsing of HTTPS GitHub URLs.""" + owner, repo, branch = GitHubRepoManager.parse_github_url("https://github.com/owner/repo") + assert owner == "owner" + assert repo == "repo" + assert branch is None + + owner, repo, branch = GitHubRepoManager.parse_github_url("https://github.com/owner/repo.git") + assert owner == "owner" + assert repo == "repo" + assert branch is None + + owner, repo, branch = GitHubRepoManager.parse_github_url("https://github.com/owner/repo/tree/main") + assert owner == "owner" + assert repo == "repo" + assert branch == "main" + + owner, repo, branch = GitHubRepoManager.parse_github_url("https://github.com/owner/repo/tree/feature/branch") + assert owner == "owner" + assert repo == "repo" + assert branch == "feature/branch" + + def test_parse_github_url_ssh_format(self): + """Test parsing of SSH GitHub URLs.""" + owner, repo, branch = GitHubRepoManager.parse_github_url("git@github.com:owner/repo.git") + assert owner == "owner" + assert repo == "repo" + assert branch is None + + owner, repo, branch = GitHubRepoManager.parse_github_url("git@github.com:owner/repo") + assert owner == "owner" + assert repo == "repo" + assert branch is None + + def test_parse_github_url_short_format(self): + """Test parsing of short GitHub identifiers.""" + owner, repo, branch = GitHubRepoManager.parse_github_url("owner/repo") + assert owner == "owner" + assert repo == "repo" + assert branch is None + + owner, repo, branch = GitHubRepoManager.parse_github_url("owner/repo@main") + assert owner == "owner" + assert repo == "repo" + assert branch == "main" + + def test_parse_github_url_invalid_formats(self): + """Test that invalid formats raise appropriate errors.""" + with pytest.raises(GitHubRepoManagerError): + GitHubRepoManager.parse_github_url("invalid-url") + + with pytest.raises(GitHubRepoManagerError): + GitHubRepoManager.parse_github_url("https://github.com/owner") + + with pytest.raises(GitHubRepoManagerError): + GitHubRepoManager.parse_github_url("owner/repo/extra") + + def test_get_cache_path(self): + """Test cache path generation.""" + with tempfile.TemporaryDirectory() as temp_dir: + manager = GitHubRepoManager(cache_dir=temp_dir) + cache_path = manager._get_cache_path("owner", "repo") # pyright: ignore[reportPrivateUsage] + expected_path = Path(temp_dir) / "owner_repo" + assert cache_path == expected_path + + def test_clone_repository_success(self, mocker: MockerFixture) -> None: + """Test successful repository cloning.""" + mock_get_env = mocker.patch("cocode.github.github_repo_manager.get_optional_env") + mock_get_env.return_value = "test_token" # Mock token to avoid GitHub CLI call + mock_run = mocker.patch("subprocess.run") + mock_run.return_value = mocker.MagicMock(returncode=0) + + with tempfile.TemporaryDirectory() as temp_dir: + manager = GitHubRepoManager(cache_dir=temp_dir) + cache_path = Path(temp_dir) / "test_repo" + + manager._clone_repository("owner", "repo", None, cache_path) # pyright: ignore[reportPrivateUsage] + + # Verify git clone command was called + mock_run.assert_called_once() + args = mock_run.call_args[0][0] + assert "git" in args + assert "clone" in args + assert "--depth=1" in args + + def test_clone_repository_failure(self, mocker: MockerFixture, suppress_error_logs: None) -> None: + """Test repository cloning failure handling.""" + from subprocess import CalledProcessError + + mock_get_env = mocker.patch("cocode.github.github_repo_manager.get_optional_env") + mock_get_env.return_value = "test_token" + mock_run = mocker.patch("subprocess.run") + mock_run.side_effect = CalledProcessError(1, "git", stderr="Repository not found") + + with tempfile.TemporaryDirectory() as temp_dir: + manager = GitHubRepoManager(cache_dir=temp_dir) + cache_path = Path(temp_dir) / "test_repo" + + with pytest.raises(GitHubRepoManagerError): + manager._clone_repository("owner", "repo", None, cache_path) # pyright: ignore[reportPrivateUsage] + + def test_get_clone_url_with_token(self, mocker: MockerFixture) -> None: + """Test clone URL generation with authentication token.""" + mock_get_env = mocker.patch("cocode.github.github_repo_manager.get_optional_env") + mock_get_env.return_value = "test_token" + + manager = GitHubRepoManager() + clone_url = manager._get_clone_url("owner", "repo") # pyright: ignore[reportPrivateUsage] # pyright: ignore[reportPrivateUsage] + + assert clone_url == "https://test_token@github.com/owner/repo.git" + + def test_get_clone_url_with_gh_cli(self, mocker: MockerFixture) -> None: + """Test clone URL generation using GitHub CLI.""" + mock_get_env = mocker.patch("cocode.github.github_repo_manager.get_optional_env") + mock_get_env.return_value = None + mock_run = mocker.patch("subprocess.run") + mock_run.return_value = mocker.MagicMock(stdout="cli_token\n", returncode=0) + + manager = GitHubRepoManager() + clone_url = manager._get_clone_url("owner", "repo") # pyright: ignore[reportPrivateUsage] + + assert clone_url == "https://cli_token@github.com/owner/repo.git" + + def test_get_clone_url_no_auth(self, mocker: MockerFixture) -> None: + """Test clone URL generation without authentication.""" + from subprocess import CalledProcessError + + mock_get_env = mocker.patch("cocode.github.github_repo_manager.get_optional_env") + mock_get_env.return_value = None + mock_run = mocker.patch("subprocess.run") + mock_run.side_effect = CalledProcessError(1, "gh") + + manager = GitHubRepoManager() + clone_url = manager._get_clone_url("owner", "repo") # pyright: ignore[reportPrivateUsage] + + assert clone_url == "https://github.com/owner/repo.git" + + def test_update_repository_success(self, mocker: MockerFixture) -> None: + """Test successful repository update.""" + mock_run = mocker.patch("subprocess.run") + mock_run.return_value = mocker.MagicMock(returncode=0) + + with tempfile.TemporaryDirectory() as temp_dir: + manager = GitHubRepoManager(cache_dir=temp_dir) + cache_path = Path(temp_dir) / "test_repo" + cache_path.mkdir() + + manager._update_repository(cache_path) # pyright: ignore[reportPrivateUsage] + + # Verify git commands were called + assert mock_run.call_count == 2 # fetch and pull + + def test_update_repository_failure(self, mocker: MockerFixture, suppress_error_logs: None) -> None: + """Test repository update failure handling.""" + from subprocess import CalledProcessError + + mock_run = mocker.patch("subprocess.run") + mock_run.side_effect = CalledProcessError(1, "git", stderr="Update failed") + + with tempfile.TemporaryDirectory() as temp_dir: + manager = GitHubRepoManager(cache_dir=temp_dir) + cache_path = Path(temp_dir) / "test_repo" + cache_path.mkdir() + + with pytest.raises(GitHubRepoManagerError): + manager._update_repository(cache_path) # pyright: ignore[reportPrivateUsage] + + def test_list_cached_repos(self): + """Test listing cached repositories.""" + with tempfile.TemporaryDirectory() as temp_dir: + manager = GitHubRepoManager(cache_dir=temp_dir) + + # Create some fake cached repos + (Path(temp_dir) / "owner1_repo1").mkdir() + (Path(temp_dir) / "owner2_repo2").mkdir() + (Path(temp_dir) / "not_a_repo_file.txt").touch() + + cached_repos = manager.list_cached_repos() + + assert "owner1/repo1" in cached_repos + assert "owner2/repo2" in cached_repos + assert len(cached_repos) == 2 + + def test_cleanup_cache(self): + """Test cache cleanup functionality.""" + import time + + with tempfile.TemporaryDirectory() as temp_dir: + manager = GitHubRepoManager(cache_dir=temp_dir) + + # Create an old directory + old_repo_dir = Path(temp_dir) / "old_repo" + old_repo_dir.mkdir() + + # Set modification time to 10 days ago + old_time = time.time() - (10 * 24 * 60 * 60) + os.utime(old_repo_dir, (old_time, old_time)) + + # Create a new directory + new_repo_dir = Path(temp_dir) / "new_repo" + new_repo_dir.mkdir() + + # Cleanup repos older than 7 days + manager.cleanup_cache(max_age_days=7) + + # Old repo should be deleted, new repo should remain + assert not old_repo_dir.exists() + assert new_repo_dir.exists() + + def test_get_local_repo_path_temp_dir(self, mocker: MockerFixture) -> None: + """Test getting local repo path with temporary directory.""" + manager = GitHubRepoManager() + + mock_clone = mocker.patch("cocode.github.github_repo_manager.GitHubRepoManager._clone_repository") + mock_mkdtemp = mocker.patch("tempfile.mkdtemp") + mock_mkdtemp.return_value = "/tmp/test_temp_dir" + + result = manager.get_local_repo_path("owner/repo", temp_dir=True) + + assert result == "/tmp/test_temp_dir" + mock_clone.assert_called_once() + + def test_get_local_repo_path_cached(self, mocker: MockerFixture) -> None: + """Test getting local repo path with existing cache.""" + mock_clone = mocker.patch("cocode.github.github_repo_manager.GitHubRepoManager._clone_repository") + mock_update = mocker.patch("cocode.github.github_repo_manager.GitHubRepoManager._update_repository") + + with tempfile.TemporaryDirectory() as temp_dir: + manager = GitHubRepoManager(cache_dir=temp_dir) + + # Create existing cache directory + cache_path = Path(temp_dir) / "owner_repo" + cache_path.mkdir() + + result = manager.get_local_repo_path("owner/repo") + + assert result == str(cache_path) + mock_update.assert_called_once() + mock_clone.assert_not_called() + + def test_get_local_repo_path_fresh_clone(self, mocker: MockerFixture) -> None: + """Test getting local repo path with fresh clone.""" + mock_clone = mocker.patch("cocode.github.github_repo_manager.GitHubRepoManager._clone_repository") + + with tempfile.TemporaryDirectory() as temp_dir: + manager = GitHubRepoManager(cache_dir=temp_dir) + + result = manager.get_local_repo_path("owner/repo") + + expected_path = str(Path(temp_dir) / "owner_repo") + assert result == expected_path + mock_clone.assert_called_once() + + def test_get_local_repo_path_force_refresh(self, mocker: MockerFixture) -> None: + """Test getting local repo path with force refresh.""" + mock_clone = mocker.patch("cocode.github.github_repo_manager.GitHubRepoManager._clone_repository") + mock_update = mocker.patch("cocode.github.github_repo_manager.GitHubRepoManager._update_repository") + + with tempfile.TemporaryDirectory() as temp_dir: + manager = GitHubRepoManager(cache_dir=temp_dir) + + # Create existing cache directory + cache_path = Path(temp_dir) / "owner_repo" + cache_path.mkdir() + + result = manager.get_local_repo_path("owner/repo", force_refresh=True) + + assert result == str(cache_path) + mock_clone.assert_called_once() + mock_update.assert_not_called() diff --git a/tests/unit/test_common.py b/tests/unit/test_common.py new file mode 100644 index 0000000..1f1d112 --- /dev/null +++ b/tests/unit/test_common.py @@ -0,0 +1,69 @@ +""" +Unit tests for common utilities. +""" + +import pytest +import typer +from pytest_mock import MockerFixture + +from cocode.common import validate_repo_path + + +class TestValidateRepoPath: + """Test cases for repository path validation.""" + + def test_validate_local_path_exists(self, mocker: MockerFixture) -> None: + """Test validation of existing local path.""" + mock_path_exists = mocker.patch("cocode.common.path_exists") + mock_path_exists.return_value = True + + result = validate_repo_path("/existing/local/path") + + # Should return absolute path + assert result.endswith("/existing/local/path") + mock_path_exists.assert_called_once() + + def test_validate_local_path_not_exists(self, mocker: MockerFixture, suppress_error_logs: None) -> None: + """Test validation of non-existing local path.""" + mock_path_exists = mocker.patch("cocode.common.path_exists") + mock_path_exists.return_value = False + + with pytest.raises(typer.Exit): + validate_repo_path("/nonexistent/local/path") + + def test_validate_github_url_success(self, mocker: MockerFixture) -> None: + """Test validation of GitHub URL with successful cloning.""" + mock_manager_class = mocker.patch("cocode.common.GitHubRepoManager") + mock_manager = mocker.MagicMock() + mock_manager.get_local_repo_path.return_value = "/tmp/cloned/repo" + mock_manager_class.return_value = mock_manager + mock_manager_class.is_github_url.return_value = True + + result = validate_repo_path("https://github.com/owner/repo") + + assert result == "/tmp/cloned/repo" + mock_manager.get_local_repo_path.assert_called_once_with("https://github.com/owner/repo", shallow=True) + + def test_validate_github_url_failure(self, mocker: MockerFixture, suppress_error_logs: None) -> None: + """Test validation of GitHub URL with cloning failure.""" + mock_manager_class = mocker.patch("cocode.common.GitHubRepoManager") + mock_manager = mocker.MagicMock() + mock_manager.get_local_repo_path.side_effect = Exception("Clone failed") + mock_manager_class.return_value = mock_manager + mock_manager_class.is_github_url.return_value = True + + with pytest.raises(typer.Exit): + validate_repo_path("https://github.com/owner/repo") + + def test_validate_short_github_format(self, mocker: MockerFixture) -> None: + """Test validation of short GitHub format.""" + mock_manager_class = mocker.patch("cocode.common.GitHubRepoManager") + mock_manager = mocker.MagicMock() + mock_manager.get_local_repo_path.return_value = "/tmp/cloned/repo" + mock_manager_class.return_value = mock_manager + mock_manager_class.is_github_url.return_value = True + + result = validate_repo_path("owner/repo") + + assert result == "/tmp/cloned/repo" + mock_manager.get_local_repo_path.assert_called_once_with("owner/repo", shallow=True) From b29dc484bd7fb920ce753e97d5b7d5e576c8310c Mon Sep 17 00:00:00 2001 From: Louis Choquel Date: Thu, 18 Sep 2025 12:55:13 +0200 Subject: [PATCH 07/10] Adapt to New inference backend config (#48) - Bump `pipelex` to `v0.10.2`: See `Pipelex` changelog [here](https://docs.pipelex.com/changelog/) --- .pipelex/inference/backends.toml | 50 +++ .pipelex/inference/backends/anthropic.toml | 67 ++++ .pipelex/inference/backends/azure_openai.toml | 69 ++++ .pipelex/inference/backends/bedrock.toml | 84 +++++ .pipelex/inference/backends/blackboxai.toml | 156 +++++++++ .pipelex/inference/backends/internal.toml | 9 + .pipelex/inference/backends/mistral.toml | 102 ++++++ .pipelex/inference/backends/ollama.toml | 31 ++ .pipelex/inference/backends/openai.toml | 124 ++++++++ .pipelex/inference/backends/perplexity.toml | 32 ++ .../inference/backends/pipelex_inference.toml | 104 ++++++ .pipelex/inference/backends/vertexai.toml | 24 ++ .pipelex/inference/backends/xai.toml | 27 ++ .../inference/deck/base_deck.toml | 56 ++-- .../inference/deck}/cocode_deck.toml | 0 .../inference/deck}/overrides.toml | 0 .pipelex/inference/routing_profiles.toml | 85 +++++ .pipelex/pipelex.toml | 21 ++ CHANGELOG.md | 4 + Makefile | 3 +- .../llm_integrations/anthropic.toml | 75 ----- .../llm_integrations/bedrock.toml | 31 -- .../llm_integrations/custom.toml | 30 -- .../llm_integrations/mistral.toml | 70 ----- .../llm_integrations/openai.toml | 153 --------- .../llm_integrations/perplexity.toml | 26 -- .../llm_integrations/vertexai.toml | 21 -- .../llm_integrations/xai.toml | 26 -- .../pipelines/base_library/documents.plx | 2 + .../pipelines/base_library/images.plx | 1 + .../base_library/meta/meta_pipeline.plx | 3 +- .../base_library/meta/pipeline_draft.py | 4 +- .../plugins/plugin_config.toml | 40 --- .../templates/base_templates.toml | 2 - pipelex.toml | 4 - pyproject.toml | 5 +- uv.lock | 296 +++++++++++------- 37 files changed, 1209 insertions(+), 628 deletions(-) create mode 100644 .pipelex/inference/backends.toml create mode 100644 .pipelex/inference/backends/anthropic.toml create mode 100644 .pipelex/inference/backends/azure_openai.toml create mode 100644 .pipelex/inference/backends/bedrock.toml create mode 100644 .pipelex/inference/backends/blackboxai.toml create mode 100644 .pipelex/inference/backends/internal.toml create mode 100644 .pipelex/inference/backends/mistral.toml create mode 100644 .pipelex/inference/backends/ollama.toml create mode 100644 .pipelex/inference/backends/openai.toml create mode 100644 .pipelex/inference/backends/perplexity.toml create mode 100644 .pipelex/inference/backends/pipelex_inference.toml create mode 100644 .pipelex/inference/backends/vertexai.toml create mode 100644 .pipelex/inference/backends/xai.toml rename cocode/pipelex_libraries/llm_deck/base_llm_deck.toml => .pipelex/inference/deck/base_deck.toml (61%) rename {cocode/pipelex_libraries/llm_deck => .pipelex/inference/deck}/cocode_deck.toml (100%) rename {cocode/pipelex_libraries/llm_deck => .pipelex/inference/deck}/overrides.toml (100%) create mode 100644 .pipelex/inference/routing_profiles.toml create mode 100644 .pipelex/pipelex.toml delete mode 100644 cocode/pipelex_libraries/llm_integrations/anthropic.toml delete mode 100644 cocode/pipelex_libraries/llm_integrations/bedrock.toml delete mode 100644 cocode/pipelex_libraries/llm_integrations/custom.toml delete mode 100644 cocode/pipelex_libraries/llm_integrations/mistral.toml delete mode 100644 cocode/pipelex_libraries/llm_integrations/openai.toml delete mode 100644 cocode/pipelex_libraries/llm_integrations/perplexity.toml delete mode 100644 cocode/pipelex_libraries/llm_integrations/vertexai.toml delete mode 100644 cocode/pipelex_libraries/llm_integrations/xai.toml delete mode 100644 cocode/pipelex_libraries/plugins/plugin_config.toml diff --git a/.pipelex/inference/backends.toml b/.pipelex/inference/backends.toml new file mode 100644 index 0000000..c3cbc68 --- /dev/null +++ b/.pipelex/inference/backends.toml @@ -0,0 +1,50 @@ +[pipelex_inference] +endpoint = "https://inference.pipelex.com/v1" +api_key = "${PIPELEX_INFERENCE_API_KEY}" + +[blackboxai] +enabled = true +endpoint = "https://api.blackbox.ai/v1" +api_key = "${BLACKBOX_API_KEY}" + +[openai] +enabled = true +api_key = "${OPENAI_API_KEY}" + +[azure_openai] +enabled = true +endpoint = "${AZURE_API_BASE}" +api_key = "${AZURE_API_KEY}" +api_version = "${AZURE_API_VERSION}" + +[anthropic] +enabled = true +api_key = "${ANTHROPIC_API_KEY}" +claude_4_tokens_limit = 8192 + +[ollama] +enabled = true +endpoint = "http://localhost:11434/v1" + +[xai] +enabled = true +endpoint = "https://api.x.ai/v1" +api_key = "${XAI_API_KEY}" + +[bedrock] +enabled = true +aws_region = "${AWS_REGION}" + +[vertexai] +enabled = true +gcp_project_id = "${GCP_PROJECT_ID}" +gcp_location = "${GCP_LOCATION}" +gcp_credentials_file_path = "${GCP_CREDENTIALS_FILE_PATH}" + +[mistral] +enabled = true +api_key = "${MISTRAL_API_KEY}" + +[internal] # software-only backend, runs internally, without AI +enabled = true + diff --git a/.pipelex/inference/backends/anthropic.toml b/.pipelex/inference/backends/anthropic.toml new file mode 100644 index 0000000..0dc8551 --- /dev/null +++ b/.pipelex/inference/backends/anthropic.toml @@ -0,0 +1,67 @@ +default_sdk = "anthropic" +default_prompting_target = "anthropic" + +[claude-3-haiku] +model_id = "claude-3-haiku-20240307" +max_tokens = 4096 +inputs = ["text", "images"] +outputs = ["text", "structured"] +max_prompt_images = 100 +costs = { input = 0.25, output = 1.25 } + +[claude-3-opus] +model_id = "claude-3-opus-20240229" +max_tokens = 4096 +inputs = ["text", "images"] +outputs = ["text", "structured"] +max_prompt_images = 100 +costs = { input = 15.0, output = 75.0 } + +["claude-3.5-sonnet"] +model_id = "claude-3-5-sonnet-20240620" +max_tokens = 8192 +inputs = ["text", "images"] +outputs = ["text", "structured"] +max_prompt_images = 100 +costs = { input = 3.0, output = 15.0 } + +["claude-3.5-sonnet-v2"] +model_id = "claude-3-5-sonnet-20241022" +max_tokens = 8192 +inputs = ["text", "images"] +outputs = ["text", "structured"] +max_prompt_images = 100 +costs = { input = 3.0, output = 15.0 } + +["claude-3.7-sonnet"] +model_id = "claude-3-7-sonnet-20250219" +max_tokens = 8192 +inputs = ["text", "images"] +outputs = ["text", "structured"] +max_prompt_images = 100 +costs = { input = 3.0, output = 15.0 } + +[claude-4-sonnet] +model_id = "claude-sonnet-4-20250514" +max_tokens = 64000 +inputs = ["text", "images"] +outputs = ["text", "structured"] +max_prompt_images = 100 +costs = { input = 3.0, output = 15.0 } + +[claude-4-opus] +model_id = "claude-opus-4-20250514" +max_tokens = 32000 +inputs = ["text", "images"] +outputs = ["text", "structured"] +max_prompt_images = 100 +costs = { input = 3.0, output = 15.0 } + +["claude-4.1-opus"] +model_id = "claude-opus-4-1-20250805" +max_tokens = 32000 +inputs = ["text", "images"] +outputs = ["text", "structured"] +max_prompt_images = 100 +costs = { input = 3.0, output = 15.0 } + diff --git a/.pipelex/inference/backends/azure_openai.toml b/.pipelex/inference/backends/azure_openai.toml new file mode 100644 index 0000000..b261788 --- /dev/null +++ b/.pipelex/inference/backends/azure_openai.toml @@ -0,0 +1,69 @@ +default_sdk = "azure_openai" +default_prompting_target = "openai" + +[gpt-4o] +model_id = "gpt-4o-2024-11-20" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 2.5, output = 10.0 } + +[gpt-4o-mini] +model_id = "gpt-4o-mini-2024-07-18" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.15, output = 0.6 } + +["gpt-4.1"] +model_id = "gpt-4.1-2025-04-14" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 2, output = 8 } + +["gpt-4.1-mini"] +model_id = "gpt-4.1-mini-2025-04-14" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.4, output = 1.6 } + +["gpt-4.1-nano"] +model_id = "gpt-4.1-nano-2025-04-14" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.1, output = 0.4 } + +[o1-mini] +model_id = "o1-mini-2024-09-12" +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 3.0, output = 12.0 } + +[o1] +model_id = "o1-2024-12-17" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 15.0, output = 60.0 } + +[o3-mini] +model_id = "o3-mini-2025-01-31" +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 1.1, output = 4.4 } + +[gpt-5-mini] +model_id = "gpt-5-mini-2025-08-07" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.25, output = 2.0 } + +[gpt-5-nano] +model_id = "gpt-5-nano-2025-08-07" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.05, output = 0.4 } + +[gpt-5-chat] +model_id = "gpt-5-chat-2025-08-07" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 1.25, output = 10.0 } + diff --git a/.pipelex/inference/backends/bedrock.toml b/.pipelex/inference/backends/bedrock.toml new file mode 100644 index 0000000..447c76f --- /dev/null +++ b/.pipelex/inference/backends/bedrock.toml @@ -0,0 +1,84 @@ +default_sdk = "bedrock_aioboto3" +default_prompting_target = "anthropic" + +[bedrock-mistral-large] +model_id = "mistral.mistral-large-2407-v1:0" +max_tokens = 8192 +inputs = ["text"] +outputs = ["text"] +costs = { input = 4.0, output = 12.0 } + +[bedrock-meta-llama-3-3-70b-instruct] +model_id = "us.meta.llama3-3-70b-instruct-v1:0" +max_tokens = 8192 +inputs = ["text"] +outputs = ["text"] +# TODO: find out the actual cost per million tokens for llama3 on bedrock +costs = { input = 3.0, output = 15.0 } + +[bedrock-nova-pro] +model_id = "us.amazon.nova-pro-v1:0" +max_tokens = 5120 +inputs = ["text"] +outputs = ["text"] +# TODO: find out the actual cost per million tokens for nova on bedrock +costs = { input = 3.0, output = 15.0 } + +################################################################################ +# Anthropic Models +################################################################################ + +["claude-3.5-sonnet"] +sdk = "bedrock_anthropic" +model_id = "us.anthropic.claude-3-5-sonnet-20240620-v1:0" +max_tokens = 8192 +inputs = ["text", "images"] +outputs = ["text", "structured"] +max_prompt_images = 100 +costs = { input = 3.0, output = 15.0 } + +["claude-3.5-sonnet-v2"] +sdk = "bedrock_anthropic" +model_id = "anthropic.claude-3-5-sonnet-20241022-v2:0" +max_tokens = 8192 +inputs = ["text", "images"] +outputs = ["text", "structured"] +max_prompt_images = 100 +costs = { input = 3.0, output = 15.0 } + +["claude-3.7-sonnet"] +sdk = "bedrock_anthropic" +model_id = "us.anthropic.claude-3-7-sonnet-20250219-v1:0" +max_tokens = 8192 +inputs = ["text", "images"] +outputs = ["text", "structured"] +max_prompt_images = 100 +costs = { input = 3.0, output = 15.0 } + +[claude-4-sonnet] +sdk = "bedrock_anthropic" +model_id = "us.anthropic.claude-sonnet-4-20250514-v1:0" +max_tokens = 64000 +inputs = ["text", "images"] +outputs = ["text", "structured"] +max_prompt_images = 100 +costs = { input = 3.0, output = 15.0 } + +[claude-4-opus] +sdk = "bedrock_anthropic" +model_id = "us.anthropic.claude-opus-4-20250514-v1:0" +max_tokens = 32000 +inputs = ["text", "images"] +outputs = ["text", "structured"] +max_prompt_images = 100 +costs = { input = 3.0, output = 15.0 } + +["claude-4.1-opus"] +sdk = "bedrock_anthropic" +model_id = "us.anthropic.claude-opus-4-1-20250805-v1:0" +max_tokens = 32000 +inputs = ["text", "images"] +outputs = ["text", "structured"] +max_prompt_images = 100 +costs = { input = 3.0, output = 15.0 } + diff --git a/.pipelex/inference/backends/blackboxai.toml b/.pipelex/inference/backends/blackboxai.toml new file mode 100644 index 0000000..14ca782 --- /dev/null +++ b/.pipelex/inference/backends/blackboxai.toml @@ -0,0 +1,156 @@ + +default_sdk = "openai" +default_prompting_target = "anthropic" + +# OpenAI Models +[gpt-4o-mini] +model_id = "blackboxai/openai/gpt-4o-mini" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.15, output = 0.60 } + +[gpt-4o] +model_id = "blackboxai/openai/gpt-4o" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 2.50, output = 10.00 } + +[o1-mini] +model_id = "blackboxai/openai/o1-mini" +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 1.10, output = 4.40 } + +[o4-mini] +model_id = "blackboxai/openai/o4-mini" +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 1.10, output = 4.40 } + +["gpt-4.5-preview"] +model_id = "blackboxai/openai/gpt-4.5-preview" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 75.00, output = 150.00 } + +# Anthropic Models +["claude-3.5-haiku"] +model_id = "blackboxai/anthropic/claude-3.5-haiku" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.80, output = 4.00 } + +["claude-3.5-sonnet"] +model_id = "blackboxai/anthropic/claude-3.5-sonnet" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 3.00, output = 15.00 } + +["claude-3.7-sonnet"] +model_id = "blackboxai/anthropic/claude-3.7-sonnet" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 3.00, output = 15.00 } + +[claude-opus-4] +model_id = "blackboxai/anthropic/claude-opus-4" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 15.00, output = 75.00 } + +[claude-sonnet-4] +model_id = "blackboxai/anthropic/claude-sonnet-4" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 3.00, output = 15.00 } + +# Google Models +["gemini-2.5-flash"] +model_id = "blackboxai/google/gemini-2.5-flash" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.30, output = 2.50 } + +["gemini-2.5-pro"] +model_id = "blackboxai/google/gemini-2.5-pro" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 1.25, output = 10.00 } + +["gemini-flash-1.5-8b"] +model_id = "blackboxai/google/gemini-flash-1.5-8b" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.04, output = 0.15 } + +# Free Models +[deepseek-chat-free] +model_id = "blackboxai/deepseek/deepseek-chat:free" +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 0.00, output = 0.00 } + +[deepseek-r1-free] +model_id = "blackboxai/deepseek/deepseek-r1:free" +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 0.00, output = 0.00 } + +["llama-3.3-70b-instruct-free"] +model_id = "blackboxai/meta-llama/llama-3.3-70b-instruct:free" +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 0.00, output = 0.00 } + +# Mistral Models +[mistral-large] +model_id = "blackboxai/mistralai/mistral-large" +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 2.00, output = 6.00 } + +[pixtral-large-2411] +model_id = "blackboxai/mistralai/pixtral-large-2411" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 2.00, output = 6.00 } + +# Cost-Effective Models +["llama-3.3-70b-instruct"] +model_id = "blackboxai/meta-llama/llama-3.3-70b-instruct" +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 0.04, output = 0.12 } + +["qwen-2.5-72b-instruct"] +model_id = "blackboxai/qwen/qwen-2.5-72b-instruct" +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 0.12, output = 0.39 } + +# Vision Models +["llama-3.2-11b-vision-instruct"] +model_id = "blackboxai/meta-llama/llama-3.2-11b-vision-instruct" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.05, output = 0.05 } + +["qwen2.5-vl-72b-instruct"] +model_id = "blackboxai/qwen/qwen2.5-vl-72b-instruct" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.25, output = 0.75 } + +# Amazon Nova Models +[nova-micro-v1] +model_id = "blackboxai/amazon/nova-micro-v1" +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 0.04, output = 0.14 } + +[nova-lite-v1] +model_id = "blackboxai/amazon/nova-lite-v1" +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 0.06, output = 0.24 } + diff --git a/.pipelex/inference/backends/internal.toml b/.pipelex/inference/backends/internal.toml new file mode 100644 index 0000000..da42bf7 --- /dev/null +++ b/.pipelex/inference/backends/internal.toml @@ -0,0 +1,9 @@ + +[pypdfium2-extract-text] +model_type = "text_extractor" +sdk = "pypdfium2" +model_id = "extract-text" +inputs = ["pdf"] +outputs = ["pages"] +costs = {} + diff --git a/.pipelex/inference/backends/mistral.toml b/.pipelex/inference/backends/mistral.toml new file mode 100644 index 0000000..b452cef --- /dev/null +++ b/.pipelex/inference/backends/mistral.toml @@ -0,0 +1,102 @@ +default_sdk = "mistral" +default_prompting_target = "mistral" + +[ministral-3b] +model_id = "ministral-3b-latest" +max_tokens = 131072 +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 0.04, output = 0.04 } + +[ministral-8b] +model_id = "ministral-8b-latest" +max_tokens = 131072 +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 0.1, output = 0.1 } + +[mistral-7b-2312] +model_id = "mistral-large-2402" +max_tokens = 32768 +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 0.25, output = 0.25 } + +[mistral-8x7b-2312] +model_id = "open-mixtral-8x7b" +max_tokens = 32768 +inputs = ["text"] +outputs = ["text"] +costs = { input = 0.7, output = 0.7 } + +[mistral-codestral-2405] +model_id = "codestral-2405" +max_tokens = 262144 +inputs = ["text"] +outputs = ["text"] +costs = { input = 1.0, output = 3.0 } + +[mistral-large-2402] +model_id = "mistral-large-2402" +max_tokens = 32768 +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 4.0, output = 12.0 } + +[mistral-large] +model_id = "mistral-large-latest" +max_tokens = 131072 +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 4.0, output = 12.0 } + +[mistral-small-2402] +model_id = "mistral-small-2402" +max_tokens = 32768 +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 1.0, output = 3.0 } + +[mistral-small] +model_id = "mistral-small-latest" +max_tokens = 32768 +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 1.0, output = 3.0 } + +[pixtral-12b] +model_id = "pixtral-12b-latest" +max_tokens = 131072 +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.15, output = 0.15 } + +[pixtral-large] +model_id = "pixtral-large-latest" +max_tokens = 131072 +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 2.0, output = 6.0 } + +[mistral-medium] +model_id = "mistral-medium-latest" +max_tokens = 128000 +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.4, output = 2.0 } + +[mistral-medium-2508] +model_id = "mistral-medium-2508" +max_tokens = 128000 +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.4, output = 2.0 } + +[mistral-ocr] +model_type = "text_extractor" +model_id = "mistral-ocr-latest" +max_tokens = 131072 +inputs = ["pdf", "image"] +outputs = ["pages"] +costs = { input = 0.4, output = 2.0 } + diff --git a/.pipelex/inference/backends/ollama.toml b/.pipelex/inference/backends/ollama.toml new file mode 100644 index 0000000..2b66dee --- /dev/null +++ b/.pipelex/inference/backends/ollama.toml @@ -0,0 +1,31 @@ +default_sdk = "openai" +default_prompting_target = "anthropic" + +[gemma3-4b] +model_id = "gemma3:4b" +inputs = ["text"] +outputs = ["text"] +max_prompt_images = 3000 +costs = { input = 0, output = 0 } + +[llama4-scout] +model_id = "llama4:scout" +inputs = ["text"] +outputs = ["text"] +max_prompt_images = 3000 +costs = { input = 0, output = 0 } + +["mistral-small3.1-24b"] +model_id = "mistral-small3.1:24b" +inputs = ["text"] +outputs = ["text"] +max_prompt_images = 3000 +costs = { input = 0, output = 0 } + +[qwen3-8b] +model_id = "qwen3:8b" +inputs = ["text"] +outputs = ["text"] +costs = { input = 0, output = 0 } +# TODO: support tokens + diff --git a/.pipelex/inference/backends/openai.toml b/.pipelex/inference/backends/openai.toml new file mode 100644 index 0000000..6e36194 --- /dev/null +++ b/.pipelex/inference/backends/openai.toml @@ -0,0 +1,124 @@ +default_sdk = "openai" +default_prompting_target = "openai" + +["gpt-3.5-turbo"] +model_id = "gpt-3.5-turbo-1106" +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 0.5, output = 1.5 } + +[gpt-4] +model_id = "gpt-4" +costs = { input = 30.0, output = 60.0 } + +[gpt-4-turbo] +model_id = "gpt-4-turbo" +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 10.0, output = 30.0 } + +[gpt-4o-2024-11-20] +model_id = "gpt-4o-2024-11-20" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 2.5, output = 10.0 } + +[gpt-4o] +model_id = "gpt-4o" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 2.5, output = 10.0 } + +[gpt-4o-mini-2024-07-18] +model_id = "gpt-4o-mini-2024-07-18" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.15, output = 0.6 } + +[gpt-4o-mini] +model_id = "gpt-4o-mini" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.15, output = 0.6 } + +["gpt-4.1"] +model_id = "gpt-4.1" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 2, output = 8 } + +["gpt-4.1-mini"] +model_id = "gpt-4.1-mini" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.4, output = 1.6 } + +["gpt-4.1-nano"] +model_id = "gpt-4.1-nano" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.1, output = 0.4 } + +[o1-mini] +model_id = "o1-mini" +inputs = ["text"] +outputs = ["text"] +costs = { input = 3.0, output = 12.0 } +constraints = ["temperature_must_be_1"] + +[o1] +model_id = "o1" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 15.0, output = 60.0 } +constraints = ["temperature_must_be_1"] + +[o3-mini] +model_id = "o3-mini" +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 1.1, output = 4.4 } +constraints = ["temperature_must_be_1"] + +[o3] +model_id = "o3" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 10.0, output = 40.0 } +constraints = ["temperature_must_be_1"] + +[o4-mini] +model_id = "o4-mini" +inputs = ["text"] +outputs = ["text", "structured"] +costs = { input = 1.1, output = 4.4 } +constraints = ["temperature_must_be_1"] + +[gpt-5] +model_id = "gpt-5" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 1.25, output = 10.0 } +constraints = ["temperature_must_be_1"] + +[gpt-5-mini] +model_id = "gpt-5-mini" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.25, output = 2.0 } +constraints = ["temperature_must_be_1"] + +[gpt-5-nano] +model_id = "gpt-5-nano" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.05, output = 0.4 } +constraints = ["temperature_must_be_1"] + +[gpt-5-chat] +model_id = "gpt-5-chat-latest" +inputs = ["text", "images"] +outputs = ["text"] +costs = { input = 1.25, output = 10.0 } +constraints = ["temperature_must_be_1"] + diff --git a/.pipelex/inference/backends/perplexity.toml b/.pipelex/inference/backends/perplexity.toml new file mode 100644 index 0000000..4eb004d --- /dev/null +++ b/.pipelex/inference/backends/perplexity.toml @@ -0,0 +1,32 @@ +default_sdk = "openai" +default_prompting_target = "anthropic" + +[sonar-pro] +model_id = "sonar-pro" +inputs = ["text"] +outputs = ["text"] + +[sonar] +model_id = "sonar" +inputs = ["text"] +outputs = ["text"] + +[sonar-deep-research] +model_id = "sonar-deep-research" +inputs = ["text"] +outputs = ["text"] + +[sonar-reasoning-pro] +model_id = "sonar-reasoning-pro" +inputs = ["text"] +outputs = ["text"] + +[sonar-reasoning] +model_id = "sonar-reasoning" +inputs = ["text"] +outputs = ["text"] + +[perplexity-deepseek-r1] +model_id = "r1-1776" +inputs = ["text"] +outputs = ["text"] diff --git a/.pipelex/inference/backends/pipelex_inference.toml b/.pipelex/inference/backends/pipelex_inference.toml new file mode 100644 index 0000000..22016ad --- /dev/null +++ b/.pipelex/inference/backends/pipelex_inference.toml @@ -0,0 +1,104 @@ +default_sdk = "openai" +default_prompting_target = "anthropic" + +# OpenAI Models +[gpt-4o-mini] +model_id = "pipelex/gpt-4o-mini" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.17, output = 0.66 } + +["gpt-4.1"] +model_id = "pipelex/gpt-4.1" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 2, output = 8 } + +["gpt-4.1-mini"] +model_id = "pipelex/gpt-4.1-mini" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.4, output = 1.6 } + +["gpt-4.1-nano"] +model_id = "pipelex/gpt-4.1-nano" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.1, output = 0.4 } + +[gpt-5-nano] +model_id = "pipelex/gpt-5-nano" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.05, output = 0.40 } + +[gpt-5-mini] +model_id = "pipelex/gpt-5-mini" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.25, output = 2.00 } + +[gpt-5-chat] +model_id = "pipelex/gpt-5-chat" +inputs = ["text", "images"] +outputs = ["text"] +costs = { input = 1.25, output = 10.00 } + +[gpt-image-1] +model_id = "pipelex/gpt-image-1" +inputs = ["text"] +outputs = ["images"] +costs = { input = 5, output = 0.01 } + +# Anthropic Models +["claude-4-sonnet"] +model_id = "pipelex/claude-4-sonnet" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 3, output = 15 } + +["claude-4.1-opus"] +model_id = "pipelex/claude-4.1-opus" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 15, output = 75 } + +# Google Models using Google API +["gemini-2.0-flash"] +model_id = "gemini/gemini-2.0-flash" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.10, output = 0.40 } + +["gemini-2.5-flash"] +model_id = "gemini/gemini-2.5-flash" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.30, output = 2.50 } + +["gemini-2.5-flash-lite"] +model_id = "gemini/gemini-2.5-flash-lite" +inputs = ["text", "images"] +outputs = ["text", "structured"] +costs = { input = 0.10, output = 0.40 } + +# XAI Models +[grok-3] +model_id = "grok-3" +inputs = ["text"] +outputs = ["text"] +costs = { input = 3, output = 15 } + +[grok-3-mini] +model_id = "grok-3-mini" +inputs = ["text"] +outputs = ["text"] +costs = { input = 0.3, output = 0.5 } + +# Flux Models +[fast-lightning-sdxl] +model_id = "pipelex/fast-lightning-sdxl" +inputs = ["text"] +outputs = ["images"] +costs = { input = 0.01, output = 0.01 } + diff --git a/.pipelex/inference/backends/vertexai.toml b/.pipelex/inference/backends/vertexai.toml new file mode 100644 index 0000000..e476cfc --- /dev/null +++ b/.pipelex/inference/backends/vertexai.toml @@ -0,0 +1,24 @@ +default_sdk = "openai" +default_prompting_target = "gemini" + +["gemini-2.0-flash"] +model_id = "google/gemini-2.0-flash" +inputs = ["text", "images"] +outputs = ["text", "structured"] +max_prompt_images = 3000 +costs = { input = 0.1, output = 0.4 } + +["gemini-2.5-pro"] +model_id = "google/gemini-2.5-pro" +inputs = ["text", "images"] +outputs = ["text", "structured"] +max_prompt_images = 3000 +costs = { input = 1.25, output = 10.0 } + +["gemini-2.5-flash"] +model_id = "google/gemini-2.5-flash" +inputs = ["text", "images"] +outputs = ["text", "structured"] +max_prompt_images = 3000 +costs = { input = 0.30, output = 2.50 } + diff --git a/.pipelex/inference/backends/xai.toml b/.pipelex/inference/backends/xai.toml new file mode 100644 index 0000000..ea8fb58 --- /dev/null +++ b/.pipelex/inference/backends/xai.toml @@ -0,0 +1,27 @@ +default_sdk = "openai" +default_prompting_target = "anthropic" + +[grok-3] +model_id = "grok-3" +inputs = ["text"] +outputs = ["text"] +costs = { input = 3, output = 15 } + +[grok-3-mini] +model_id = "grok-3-mini" +inputs = ["text"] +outputs = ["text"] +costs = { input = 0.3, output = 0.5 } + +[grok-3-fast] +model_id = "grok-3-fast-latest" +inputs = ["text"] +outputs = ["text"] +costs = { input = 5, output = 25 } + +[grok-3-mini-fast] +model_id = "grok-3-mini-fast-latest" +inputs = ["text"] +outputs = ["text"] +costs = { input = 0.15, output = 4 } + diff --git a/cocode/pipelex_libraries/llm_deck/base_llm_deck.toml b/.pipelex/inference/deck/base_deck.toml similarity index 61% rename from cocode/pipelex_libraries/llm_deck/base_llm_deck.toml rename to .pipelex/inference/deck/base_deck.toml index 726851f..51d646c 100644 --- a/cocode/pipelex_libraries/llm_deck/base_llm_deck.toml +++ b/.pipelex/inference/deck/base_deck.toml @@ -1,23 +1,20 @@ -# LLM Deck base +# Base model deck #################################################################################################### -# LLM Handles +# Aliases #################################################################################################### -llm_external_handles = [] - -# Match a llm_handle with either just and llm_name -# or a complete blueprint including llm_version (defaulting to "latest") and llm_platform_choice (defaulting to "default"). -[llm_handles] -gpt-4o-2024-11-20 = { llm_name = "gpt-4o", llm_version = "2024-11-20" } +[aliases] base-claude = "claude-4-sonnet" base-gpt = "gpt-5" +base-gemini = "gemini-2.5-flash" +base-mistral = "mistral-medium" best-gpt = "gpt-5" -best-claude = "claude-4-1-opus" +best-claude = "claude-4.1-opus" best-gemini = "gemini-2.5-pro" -best-mistral = "mistral-large" +best-mistral = "mistral-medium" best-grok = "grok-3" cheap_llm_for_text = "gpt-4o-mini" @@ -26,6 +23,8 @@ cheap_llm_for_object = "gpt-4o-mini" base_llm_for_text = "gpt-4o-mini" base_llm_for_object = "gpt-4o-mini" +llm_to_engineer = ["claude-4.1-opus", "gemini-2.5-pro"] +llm_for_large_codebase = ["gemini-2.5-pro", "claude-4-sonnet", "gpt-5"] #################################################################################################### # LLM Presets @@ -47,40 +46,41 @@ llm_for_testing_gen_object = { llm_handle = "base_llm_for_object", temperature = #################################################################################################### # LLM Presets — Specific skills +# Cheap llm +llm_cheap = { llm_handle = "base-gemini", temperature = 0.1 } + # Generation skills llm_for_factual_writing = { llm_handle = "base-gpt", temperature = 0.1 } -llm_for_creative_writing = { llm_handle = "best-claude", temperature = 0.9 } +llm_for_creative_writing = { llm_handle = "claude-4-sonnet", temperature = 0.9 } # Reasoning skills llm_to_reason_short = { llm_handle = "base-claude", temperature = 0.5, max_tokens = 500 } llm_to_reason = { llm_handle = "base-claude", temperature = 1 } -llm_to_reason_on_diagram = { llm_handle = "best-claude", temperature = 0.5 } +llm_to_reason_on_diagram = { llm_handle = "base-claude", temperature = 0.5 } # Search and answer skills -llm_to_answer = { llm_handle = "best-claude", temperature = 0.1 } -llm_to_retrieve = { llm_handle = "best-gemini", temperature = 0.1 } +llm_to_answer = { llm_handle = "base-claude", temperature = 0.1 } +llm_to_retrieve = { llm_handle = "base-gemini", temperature = 0.1 } llm_for_enrichment = { llm_handle = "gpt-4o", temperature = 0.1 } -llm_to_enrich = { llm_handle = "best-claude", temperature = 0.1 } +llm_to_enrich = { llm_handle = "base-claude", temperature = 0.1 } llm_for_question_and_excerpt_reformulation = { llm_handle = "gpt-4o", temperature = 0.9 } # Engineering skills -llm_to_engineer = { llm_handle = "best-claude", temperature = 0.2 } -llm_to_pipelex = { llm_handle = "best-claude", temperature = 0.2 } -# llm_to_pipelex = { llm_handle = "gpt-5-mini", temperature = 0.2 } +llm_to_engineer = { llm_handle = "base-claude", temperature = 0.2 } +llm_to_pipelex = { llm_handle = "base-claude", temperature = 0.2 } # Image skills -llm_to_write_imgg_prompt = { llm_handle = "best-claude", temperature = 0.2 } -llm_to_describe_img = { llm_handle = "best-claude", temperature = 0.5 } -llm_to_design_fashion = { llm_handle = "best-claude", temperature = 0.7 } -llm_for_img_to_text = { llm_handle = "best-claude", temperature = 0.1 } +llm_to_write_imgg_prompt = { llm_handle = "base-claude", temperature = 0.2 } +llm_to_describe_img = { llm_handle = "base-claude", temperature = 0.5 } +llm_to_design_fashion = { llm_handle = "base-claude", temperature = 0.7 } +llm_for_img_to_text = { llm_handle = "base-claude", temperature = 0.1 } # Extraction skills -llm_to_extract_diagram = { llm_handle = "best-claude", temperature = 0.5 } -llm_to_extract_invoice = { llm_handle = "claude-3-7-sonnet", temperature = 0.1 } -llm_to_extract_invoice_from_scan = { llm_handle = "best-claude", temperature = 0.5 } -llm_to_extract_legal_terms = { llm_handle = "best-claude", temperature = 0.1 } -llm_to_extract_tables = { llm_handle = "best-claude", temperature = 0.1 } - +llm_to_extract_diagram = { llm_handle = "base-claude", temperature = 0.5 } +llm_to_extract_invoice = { llm_handle = "claude-4-sonnet", temperature = 0.1 } +llm_to_extract_invoice_from_scan = { llm_handle = "base-claude", temperature = 0.5 } +llm_to_extract_legal_terms = { llm_handle = "base-claude", temperature = 0.1 } +llm_to_extract_tables = { llm_handle = "base-claude", temperature = 0.1 } #################################################################################################### # LLM Choices diff --git a/cocode/pipelex_libraries/llm_deck/cocode_deck.toml b/.pipelex/inference/deck/cocode_deck.toml similarity index 100% rename from cocode/pipelex_libraries/llm_deck/cocode_deck.toml rename to .pipelex/inference/deck/cocode_deck.toml diff --git a/cocode/pipelex_libraries/llm_deck/overrides.toml b/.pipelex/inference/deck/overrides.toml similarity index 100% rename from cocode/pipelex_libraries/llm_deck/overrides.toml rename to .pipelex/inference/deck/overrides.toml diff --git a/.pipelex/inference/routing_profiles.toml b/.pipelex/inference/routing_profiles.toml new file mode 100644 index 0000000..be9b51d --- /dev/null +++ b/.pipelex/inference/routing_profiles.toml @@ -0,0 +1,85 @@ +# Routing profile library - Routes models to their backends +# ================================================ +# This file controls which backend serves which model. +# Simply change the 'active' field to switch profiles, +# or you can add your own custom profiles. +# ================================================ + +# Which profile to use (change this to switch routing) +active = "all_pipelex" + +# We recommend using the "all_pipelex" profile to get a head start with all models. +# The Pipelex Inference backend is currently not recommended for production use, +# but it's great for development and testing. +# To use the Pipelex Inference backend (all_pipelex profile): +# 1. Join our Discord community to get your free API key (no credit card required): +# Visit https://go.pipelex.com/discord and request your key in the appropriate channel +# 2. Set the environment variable: export PIPELEX_INFERENCE_API_KEY="your-api-key" +# 3. The .pipelex/inference/backends.toml is already configured with api_key = "${PIPELEX_INFERENCE_API_KEY}" +# which will get the key from the environment variable. + +# ============================================ +# Routing Profiles +# ============================================ + +[profiles.all_pipelex] +description = "Use Pipelex Inference backend for all models" +default = "pipelex_inference" + +[profiles.all_anthropic] +description = "Use Anthropic backend for all its supported models" +default = "anthropic" + +[profiles.all_azure_openai] +description = "Use Azure OpenAI backend for all its supported models" +default = "azure_openai" + +[profiles.to_test_various_backends] +description = "Route models to various backends" + +[profiles.custom_routing] +description = "Custom routing" +default = "pipelex_inference" + +[profiles.custom_routing.routes] +# Pattern matching: "model-pattern" = "backend-name" +"gpt-*" = "pipelex_inference" +"claude-*" = "pipelex_inference" +"gemini-*" = "pipelex_inference" +"grok-*" = "pipelex_inference" +"*-sdxl" = "pipelex_inference" + +[profiles.to_test_various_backends.routes] +"gpt-5-nano" = "pipelex_inference" +"gpt-4o-mini" = "blackboxai" +"gpt-5-mini" = "openai" +"gpt-5-chat" = "azure_openai" + +"claude-4-sonnet" = "pipelex_inference" +"claude-3.5-sonnet" = "bedrock" +"claude-3.5-sonnet-v2" = "anthropic" +"claude-3.7-sonnet" = "blackboxai" + +"gemini-2.5-flash-lite" = "pipelex_inference" +"gemini-2.5-flash" = "blackboxai" +"gemini-2.5-pro" = "vertexai" + +"grok-3" = "pipelex_inference" +"grok-3-mini" = "xai" + +# ============================================ +# Custom Profiles +# ============================================ +# Add your own profiles below following the same pattern: +# +# [profiles.your_profile_name] +# description = "What this profile does" +# default = "backend-name" # Where to route models by default +# [profiles.your_profile_name.routes] +# "model-pattern" = "backend-name" # Specific routing rules +# +# Pattern matching supports: +# - Exact names: "gpt-4o-mini" +# - Wildcards: "claude-*" (matches all models starting with claude-) +# - Partial wildcards: "*-sonnet" (matches all sonnet variants) + diff --git a/.pipelex/pipelex.toml b/.pipelex/pipelex.toml new file mode 100644 index 0000000..b8f8b12 --- /dev/null +++ b/.pipelex/pipelex.toml @@ -0,0 +1,21 @@ +[pipelex.aws_config] +api_key_method = "env" +# The possible values are "env" and "secret_provider". +# "env" means means that the env var are stored in your .env file. +# "secret_provider" means that the env var are stored in your Secret Manager (See the doc for injecting a secret provider). + +[cogt] + +#################################################################################################### +# OCR config +#################################################################################################### + +[cogt.ocr_config] +page_output_text_file_name = "page_text.md" + +[pipelex.feature_config] +# WIP/Experimental feature flags +is_pipeline_tracking_enabled = false +is_activity_tracking_enabled = false +is_reporting_enabled = true + diff --git a/CHANGELOG.md b/CHANGELOG.md index 92e7114..8427cdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [v0.2.2] - 2025-09-18 + +- Bump `pipelex` to `v0.10.2`: See `Pipelex` changelog [here](https://docs.pipelex.com/changelog/) + ## [v0.2.1] - 2025-09-06 ### Changed diff --git a/Makefile b/Makefile index 46271d8..0f5f8ee 100644 --- a/Makefile +++ b/Makefile @@ -138,9 +138,8 @@ env: check-uv install: env $(call PRINT_TITLE,"Installing dependencies") @. $(VIRTUAL_ENV)/bin/activate && \ - uv sync --all-extras && \ + uv sync --all-extras --no-cache && \ echo "Installed dependencies in ${VIRTUAL_ENV}"; - install-latest: env $(call PRINT_TITLE,"Installing dependencies with latest versions") @. $(VIRTUAL_ENV)/bin/activate && \ diff --git a/cocode/pipelex_libraries/llm_integrations/anthropic.toml b/cocode/pipelex_libraries/llm_integrations/anthropic.toml deleted file mode 100644 index 079fbcc..0000000 --- a/cocode/pipelex_libraries/llm_integrations/anthropic.toml +++ /dev/null @@ -1,75 +0,0 @@ - - -[claude-3.claude-3-haiku.latest] -max_tokens = 4096 -is_gen_object_supported = true -is_vision_supported = true -max_prompt_images = 100 -cost_per_million_tokens_usd = { input = 0.25, output = 1.25 } -platform_llm_id = { anthropic = "claude-3-haiku-20240307" } - -[claude-3.claude-3-opus.latest] -max_tokens = 4096 -is_gen_object_supported = true -is_vision_supported = true -max_prompt_images = 100 -cost_per_million_tokens_usd = { input = 15.0, output = 75.0 } -platform_llm_id = { anthropic = "claude-3-opus-20240229" } - -["claude-3.5".claude-3-5-sonnet.latest] -max_tokens = 8192 -is_gen_object_supported = true -is_vision_supported = true -max_prompt_images = 100 -cost_per_million_tokens_usd = { input = 3.0, output = 15.0 } -platform_llm_id = { anthropic = "claude-3-5-sonnet-20240620", bedrock_anthropic = "us.anthropic.claude-3-5-sonnet-20240620-v1:0" } -default_platform = "anthropic" - -["claude-3.5".claude-3-5-sonnet-v2.latest] -max_tokens = 8192 -is_gen_object_supported = true -is_vision_supported = true -max_prompt_images = 100 -cost_per_million_tokens_usd = { input = 3.0, output = 15.0 } -platform_llm_id = { anthropic = "claude-3-5-sonnet-20241022", bedrock_anthropic = "anthropic.claude-3-5-sonnet-20241022-v2:0" } -default_platform = "anthropic" - -["claude-3.7".claude-3-7-sonnet.latest] -max_tokens = 8192 -is_gen_object_supported = true -is_vision_supported = true -max_prompt_images = 100 -cost_per_million_tokens_usd = { input = 3.0, output = 15.0 } -platform_llm_id = { anthropic = "claude-3-7-sonnet-20250219", bedrock_anthropic = "us.anthropic.claude-3-7-sonnet-20250219-v1:0" } -default_platform = "anthropic" - - -[claude-4.claude-4-sonnet.latest] -max_tokens = 64000 -is_gen_object_supported = true -is_vision_supported = true -max_prompt_images = 100 -cost_per_million_tokens_usd = { input = 3.0, output = 15.0 } -platform_llm_id = { anthropic = "claude-sonnet-4-20250514", bedrock_anthropic = "us.anthropic.claude-sonnet-4-20250514-v1:0" } -default_platform = "anthropic" - - -[claude-4.claude-4-opus.latest] -max_tokens = 32000 -is_gen_object_supported = true -is_vision_supported = true -max_prompt_images = 100 -cost_per_million_tokens_usd = { input = 3.0, output = 15.0 } -platform_llm_id = { anthropic = "claude-opus-4-20250514", bedrock_anthropic = "us.anthropic.claude-opus-4-20250514-v1:0" } -default_platform = "anthropic" - - -["claude-4.1".claude-4-1-opus.latest] -max_tokens = 32000 -is_gen_object_supported = true -is_vision_supported = true -max_prompt_images = 100 -cost_per_million_tokens_usd = { input = 3.0, output = 15.0 } -platform_llm_id = { anthropic = "claude-opus-4-1-20250805", bedrock_anthropic = "us.anthropic.claude-opus-4-1-20250805-v1:0" } -default_platform = "anthropic" - diff --git a/cocode/pipelex_libraries/llm_integrations/bedrock.toml b/cocode/pipelex_libraries/llm_integrations/bedrock.toml deleted file mode 100644 index bf44043..0000000 --- a/cocode/pipelex_libraries/llm_integrations/bedrock.toml +++ /dev/null @@ -1,31 +0,0 @@ - - -[bedrock-mistral-large.bedrock-mistral-large.latest] -max_tokens = 8192 -is_gen_object_supported = false -cost_per_million_tokens_usd = { input = 4.0, output = 12.0 } -platform_llm_id = { bedrock = "mistral.mistral-large-2407-v1:0" } - - -[bedrock-anthropic-claude.bedrock-claude-3-7-sonnet.latest] -max_tokens = 8192 -is_gen_object_supported = false -cost_per_million_tokens_usd = { input = 3.0, output = 15.0 } -platform_llm_id = { bedrock = "us.anthropic.claude-3-7-sonnet-20250219-v1:0" } - - -[bedrock-meta-llama-3.bedrock-meta-llama-3-3-70b-instruct.latest] -max_tokens = 8192 -is_gen_object_supported = false -# TODO: find out the actual cost per million tokens for llama3 on bedrock -cost_per_million_tokens_usd = { input = 3.0, output = 15.0 } -platform_llm_id = { bedrock = "us.meta.llama3-3-70b-instruct-v1:0" } - - -[bedrock-amazon-nova.bedrock-nova-pro.latest] -max_tokens = 5120 -is_gen_object_supported = false -# TODO: find out the actual cost per million tokens for nova on bedrock -cost_per_million_tokens_usd = { input = 3.0, output = 15.0 } -platform_llm_id = { bedrock = "us.amazon.nova-pro-v1:0" } - diff --git a/cocode/pipelex_libraries/llm_integrations/custom.toml b/cocode/pipelex_libraries/llm_integrations/custom.toml deleted file mode 100644 index ecf6d74..0000000 --- a/cocode/pipelex_libraries/llm_integrations/custom.toml +++ /dev/null @@ -1,30 +0,0 @@ - - -[custom-gemma-3."gemma3:4b".latest] -is_gen_object_supported = false -is_vision_supported = true -max_prompt_images = 3000 -cost_per_million_tokens_usd = { input = 0, output = 0 } -platform_llm_id = { custom_llm = "gemma3:4b" } - -[custom-llama-4."llama4:scout".latest] -is_gen_object_supported = false -is_vision_supported = true -max_prompt_images = 3000 -cost_per_million_tokens_usd = { input = 0, output = 0 } -platform_llm_id = { custom_llm = "llama4:scout" } - -["custom-mistral-small3.1"."mistral-small3.1".latest] -is_gen_object_supported = false -is_vision_supported = true -max_prompt_images = 3000 -cost_per_million_tokens_usd = { input = 0, output = 0 } -platform_llm_id = { custom_llm = "mistral-small3.1:24b" } - -["custom-qwen3"."qwen3:8b".latest] -is_gen_object_supported = false -is_vision_supported = false -cost_per_million_tokens_usd = { input = 0, output = 0 } -platform_llm_id = { custom_llm = "qwen3:8b" } -# TODO: support tokens - diff --git a/cocode/pipelex_libraries/llm_integrations/mistral.toml b/cocode/pipelex_libraries/llm_integrations/mistral.toml deleted file mode 100644 index c99cdd8..0000000 --- a/cocode/pipelex_libraries/llm_integrations/mistral.toml +++ /dev/null @@ -1,70 +0,0 @@ - - -[ministral.ministral-3b.latest] -max_tokens = 131072 -is_gen_object_supported = true -cost_per_million_tokens_usd = { input = 0.04, output = 0.04 } -platform_llm_id = { mistral = "ministral-3b-latest" } - -[ministral.ministral-8b.latest] -max_tokens = 131072 -is_gen_object_supported = true -cost_per_million_tokens_usd = { input = 0.1, output = 0.1 } -platform_llm_id = { mistral = "ministral-8b-latest" } - -[mistral-7b.mistral-7b."2312"] -max_tokens = 32768 -is_gen_object_supported = true -cost_per_million_tokens_usd = { input = 0.25, output = 0.25 } -platform_llm_id = { mistral = "mistral-large-2402" } - -[mistral-8x7b.mistral-8x7b."2312"] -max_tokens = 32768 -is_gen_object_supported = false -cost_per_million_tokens_usd = { input = 0.7, output = 0.7 } -platform_llm_id = { mistral = "open-mixtral-8x7b" } - -[mistral-codestral.mistral-codestral."2405"] -max_tokens = 262144 -is_gen_object_supported = false -cost_per_million_tokens_usd = { input = 1.0, output = 3.0 } -platform_llm_id = { mistral = "codestral-2405" } - -[mistral-large.mistral-large."2402"] -max_tokens = 32768 -is_gen_object_supported = true -cost_per_million_tokens_usd = { input = 4.0, output = 12.0 } -platform_llm_id = { mistral = "mistral-large-2402" } - -[mistral-large.mistral-large.latest] -max_tokens = 131072 -is_gen_object_supported = true -cost_per_million_tokens_usd = { input = 4.0, output = 12.0 } -platform_llm_id = { mistral = "mistral-large-latest" } - -[mistral-small.mistral-small."2402"] -max_tokens = 32768 -is_gen_object_supported = true -cost_per_million_tokens_usd = { input = 1.0, output = 3.0 } -platform_llm_id = { mistral = "mistral-small-2402" } - -[mistral-small.mistral-small.latest] -max_tokens = 32768 -is_gen_object_supported = true -cost_per_million_tokens_usd = { input = 1.0, output = 3.0 } -platform_llm_id = { mistral = "mistral-small-latest" } - -[pixtral.pixtral-12b.latest] -max_tokens = 131072 -is_gen_object_supported = true -is_vision_supported = true -cost_per_million_tokens_usd = { input = 0.15, output = 0.15 } -platform_llm_id = { mistral = "pixtral-12b-latest" } - -[pixtral.pixtral-large.latest] -max_tokens = 131072 -is_gen_object_supported = true -is_vision_supported = true -cost_per_million_tokens_usd = { input = 2.0, output = 6.0 } -platform_llm_id = { mistral = "pixtral-large-latest" } - diff --git a/cocode/pipelex_libraries/llm_integrations/openai.toml b/cocode/pipelex_libraries/llm_integrations/openai.toml deleted file mode 100644 index 895e8c3..0000000 --- a/cocode/pipelex_libraries/llm_integrations/openai.toml +++ /dev/null @@ -1,153 +0,0 @@ -["gpt-3.5"."gpt-3.5-turbo".latest] -is_gen_object_supported = true -cost_per_million_tokens_usd = { input = 0.5, output = 1.5 } -platform_llm_id = { openai = "gpt-3.5-turbo-1106" } -default_platform = "openai" - -[gpt-4.gpt-4.latest] -is_gen_object_supported = false -is_vision_supported = false -cost_per_million_tokens_usd = { input = 30.0, output = 60.0 } -platform_llm_id = { openai = "gpt-4" } - -[gpt-4.gpt-4-turbo.0125-preview] -is_gen_object_supported = true -is_vision_supported = false -cost_per_million_tokens_usd = { input = 10.0, output = 30.0 } -platform_llm_id = { openai = "gpt-4-0125-preview" } - -[gpt-4.gpt-4-turbo.1106-preview] -is_gen_object_supported = true -is_vision_supported = false -cost_per_million_tokens_usd = { input = 10.0, output = 30.0 } -platform_llm_id = { openai = "gpt-4-1106-preview" } - -[gpt-4.gpt-4-turbo."2024-04-09"] -is_gen_object_supported = true -is_vision_supported = false -cost_per_million_tokens_usd = { input = 10.0, output = 30.0 } -platform_llm_id = { openai = "gpt-4-turbo-2024-04-09" } - -[gpt-4.gpt-4-turbo.latest] -is_gen_object_supported = true -is_vision_supported = false -cost_per_million_tokens_usd = { input = 10.0, output = 30.0 } -platform_llm_id = { openai = "gpt-4-turbo" } - -[gpt-4o.gpt-4o."2024-05-13"] -is_gen_object_supported = true -is_vision_supported = true -cost_per_million_tokens_usd = { input = 5.0, output = 15.0 } -platform_llm_id = { openai = "gpt-4o-2024-05-13" } - -[gpt-4.gpt-4o."2024-08-06"] -is_gen_object_supported = true -is_vision_supported = true -cost_per_million_tokens_usd = { input = 2.5, output = 10.0 } -platform_llm_id = { openai = "gpt-4o-2024-08-06" } - -[gpt-4.gpt-4o."2024-11-20"] -is_gen_object_supported = true -is_vision_supported = true -cost_per_million_tokens_usd = { input = 2.5, output = 10.0 } -platform_llm_id = { azure_openai = "gpt-4o-2024-11-20", openai = "gpt-4o-2024-11-20" } -default_platform = "openai" - -[gpt-4o.gpt-4o.latest] -is_gen_object_supported = true -is_vision_supported = true -cost_per_million_tokens_usd = { input = 2.5, output = 10.0 } -platform_llm_id = { azure_openai = "gpt-4o-2024-11-20", openai = "gpt-4o" } -default_platform = "openai" - -[gpt-4o.gpt-4o-mini."2024-07-18"] -is_gen_object_supported = true -is_vision_supported = true -cost_per_million_tokens_usd = { input = 0.15, output = 0.6 } -platform_llm_id = { azure_openai = "gpt-4o-mini-2024-07-18", openai = "gpt-4o-mini-2024-07-18" } -default_platform = "openai" - -[gpt-4o.gpt-4o-mini.latest] -is_gen_object_supported = true -is_vision_supported = true -cost_per_million_tokens_usd = { input = 0.15, output = 0.6 } -platform_llm_id = { azure_openai = "gpt-4o-mini-2024-07-18", openai = "gpt-4o-mini" } -default_platform = "openai" - -["gpt-4.1"."gpt-4.1".latest] -is_gen_object_supported = true -is_vision_supported = true -cost_per_million_tokens_usd = { input = 2, output = 8 } -platform_llm_id = { azure_openai = "gpt-4.1-2025-04-14", openai = "gpt-4.1" } -default_platform = "openai" - -["gpt-4.1"."gpt-4.1-mini".latest] -is_gen_object_supported = true -is_vision_supported = true -cost_per_million_tokens_usd = { input = 0.4, output = 1.6 } -platform_llm_id = { azure_openai = "gpt-4.1-mini-2025-04-14", openai = "gpt-4.1-mini" } -default_platform = "openai" - -["gpt-4.1"."gpt-4.1-nano".latest] -is_gen_object_supported = true -is_vision_supported = true -cost_per_million_tokens_usd = { input = 0.1, output = 0.4 } -platform_llm_id = { azure_openai = "gpt-4.1-nano-2025-04-14", openai = "gpt-4.1-nano" } -default_platform = "openai" - -[o.o1-mini.latest] -is_gen_object_supported = false -is_vision_supported = false -cost_per_million_tokens_usd = { input = 3.0, output = 12.0 } -platform_llm_id = { azure_openai = "o1-mini-2024-09-12", openai = "o1-mini" } - -[o.o1.latest] -is_gen_object_supported = true -is_vision_supported = true -cost_per_million_tokens_usd = { input = 15.0, output = 60.0 } -platform_llm_id = { azure_openai = "o1-2024-12-17", openai = "o1" } - -[o.o3-mini.latest] -is_gen_object_supported = true -cost_per_million_tokens_usd = { input = 1.1, output = 4.4 } -platform_llm_id = { azure_openai = "o3-mini-2025-01-31", openai = "o3-mini" } -default_platform = "openai" - -[o.o3.latest] -is_gen_object_supported = true -is_vision_supported = true -cost_per_million_tokens_usd = { input = 10.0, output = 40.0 } -platform_llm_id = { openai = "o3" } - -[o.o4-mini.latest] -is_gen_object_supported = true -cost_per_million_tokens_usd = { input = 1.1, output = 4.4 } -platform_llm_id = { openai = "o4-mini" } - -[gpt-5.gpt-5.latest] -is_gen_object_supported = true -is_vision_supported = true -cost_per_million_tokens_usd = { input = 1.25, output = 10.0 } -platform_llm_id = { openai = "gpt-5" } - -[gpt-5.gpt-5-mini.latest] -is_gen_object_supported = true -is_vision_supported = true -cost_per_million_tokens_usd = { input = 0.25, output = 2.0 } -platform_llm_id = { azure_openai = "gpt-5-mini-2025-08-07", openai = "gpt-5-mini" } -default_platform = "openai" - -[gpt-5.gpt-5-nano.latest] -is_gen_object_supported = true -is_vision_supported = true -cost_per_million_tokens_usd = { input = 0.05, output = 0.4 } -platform_llm_id = { azure_openai = "gpt-5-nano-2025-08-07", openai = "gpt-5-nano" } -default_platform = "openai" - -[gpt-5-chat.gpt-5-chat.latest] -is_gen_object_supported = false -is_vision_supported = true -cost_per_million_tokens_usd = { input = 1.25, output = 10.0 } -platform_llm_id = { azure_openai = "gpt-5-chat-2025-08-07", openai = "gpt-5-chat-latest" } -default_platform = "openai" - diff --git a/cocode/pipelex_libraries/llm_integrations/perplexity.toml b/cocode/pipelex_libraries/llm_integrations/perplexity.toml deleted file mode 100644 index fb681f7..0000000 --- a/cocode/pipelex_libraries/llm_integrations/perplexity.toml +++ /dev/null @@ -1,26 +0,0 @@ - - -[perplexity-search.sonar-pro.latest] -is_gen_object_supported = false -platform_llm_id = { perplexity = "sonar-pro" } - -[perplexity-search.sonar.latest] -is_gen_object_supported = false -platform_llm_id = { perplexity = "sonar" } - -[perplexity-research.sonar-deep-research.latest] -is_gen_object_supported = false -platform_llm_id = { perplexity = "sonar-deep-research" } - -[perplexity-reasoning.sonar-reasoning-pro.latest] -is_gen_object_supported = false -platform_llm_id = { perplexity = "sonar-reasoning-pro" } - -[perplexity-reasoning.sonar-reasoning.latest] -is_gen_object_supported = false -platform_llm_id = { perplexity = "sonar-reasoning" } - -[perplexity-deepseek.perplexity-deepseek-r1.latest] -is_gen_object_supported = false -platform_llm_id = { perplexity = "r1-1776" } - diff --git a/cocode/pipelex_libraries/llm_integrations/vertexai.toml b/cocode/pipelex_libraries/llm_integrations/vertexai.toml deleted file mode 100644 index 8b6d3a7..0000000 --- a/cocode/pipelex_libraries/llm_integrations/vertexai.toml +++ /dev/null @@ -1,21 +0,0 @@ -[gemini."gemini-2.0-flash".latest] -is_gen_object_supported = true -is_vision_supported = true -max_prompt_images = 3000 -cost_per_million_tokens_usd = { input = 0.1, output = 0.4 } -platform_llm_id = { vertexai = "google/gemini-2.0-flash" } - -[gemini."gemini-2.5-pro"."latest"] -is_gen_object_supported = true -is_vision_supported = true -max_prompt_images = 3000 -cost_per_million_tokens_usd = { input = 0.0, output = 0.0 } -platform_llm_id = { vertexai = "google/gemini-2.5-pro" } - -[gemini."gemini-2.5-flash".latest] -is_gen_object_supported = true -is_vision_supported = true -max_prompt_images = 3000 -cost_per_million_tokens_usd = { input = 0.15, output = 0.6 } -platform_llm_id = { vertexai = "google/gemini-2.5-flash" } - diff --git a/cocode/pipelex_libraries/llm_integrations/xai.toml b/cocode/pipelex_libraries/llm_integrations/xai.toml deleted file mode 100644 index 31393cd..0000000 --- a/cocode/pipelex_libraries/llm_integrations/xai.toml +++ /dev/null @@ -1,26 +0,0 @@ - -[grok-3.grok-3.latest] -is_gen_object_supported = true -is_vision_supported = false -cost_per_million_tokens_usd = { input = 3, output = 15 } -platform_llm_id = { xai = "grok-3-latest" } - -[grok-3.grok-3-mini.latest] -is_gen_object_supported = true -is_vision_supported = false -cost_per_million_tokens_usd = { input = 0.3, output = 0.5 } -platform_llm_id = { xai = "grok-3-mini-latest" } - -[grok-3.grok-3-fast.latest] -is_gen_object_supported = true -is_vision_supported = false -cost_per_million_tokens_usd = { input = 5, output = 25 } -platform_llm_id = { xai = "grok-3-fast-latest" } - - -[grok-3.grok-3-mini-fast.latest] -is_gen_object_supported = true -is_vision_supported = false -cost_per_million_tokens_usd = { input = 0.15, output = 4 } -platform_llm_id = { xai = "grok-3-mini-fast-latest" } - diff --git a/cocode/pipelex_libraries/pipelines/base_library/documents.plx b/cocode/pipelex_libraries/pipelines/base_library/documents.plx index c3826d8..0f8b290 100644 --- a/cocode/pipelex_libraries/pipelines/base_library/documents.plx +++ b/cocode/pipelex_libraries/pipelines/base_library/documents.plx @@ -17,6 +17,7 @@ inputs = { ocr_input = "PDF" } output = "Page" page_images = true page_views = false +ocr_model = "mistral-ocr" [pipe.ocr_page_contents_and_views_from_pdf] type = "PipeOcr" @@ -25,4 +26,5 @@ inputs = { ocr_input = "PDF" } output = "Page" page_images = true page_views = true +ocr_model = "mistral-ocr" diff --git a/cocode/pipelex_libraries/pipelines/base_library/images.plx b/cocode/pipelex_libraries/pipelines/base_library/images.plx index b46d086..55d5ea4 100644 --- a/cocode/pipelex_libraries/pipelines/base_library/images.plx +++ b/cocode/pipelex_libraries/pipelines/base_library/images.plx @@ -25,6 +25,7 @@ inputs = { image = "Image" } output = "VisualDescription" system_prompt = "You are a very good observer." llm = "llm_to_describe_img" +structuring_method = "preliminary_text" prompt_template = """ Describe the provided image in great detail. """ diff --git a/cocode/pipelex_libraries/pipelines/base_library/meta/meta_pipeline.plx b/cocode/pipelex_libraries/pipelines/base_library/meta/meta_pipeline.plx index cbeb7ad..791c639 100644 --- a/cocode/pipelex_libraries/pipelines/base_library/meta/meta_pipeline.plx +++ b/cocode/pipelex_libraries/pipelines/base_library/meta/meta_pipeline.plx @@ -10,7 +10,7 @@ PipeBlueprint = "Details enabling to create a pipe" [concept.PipelexBundleBlueprint] definition = "Details enabling to create a pipeline" -structure = "PipelexBundleBlueprintStuff" +structure = "PipelexBundleBlueprint" [pipe] [pipe.build_blueprint] @@ -51,7 +51,6 @@ definition = "Generate a pipeline blueprint from natural language requirements" inputs = { build_pipeline_rules = "Text", pipeline_draft = "Text", pipeline_name = "Text", domain = "Text", requirements = "Text" } output = "PipelexBundleBlueprint" llm = "llm_to_pipelex" -# structuring_method = "preliminary_text" prompt_template = """ You are a Pipelex pipeline architect. Build a structured pipeline blueprint from the provided pipeline draft text. diff --git a/cocode/pipelex_libraries/pipelines/base_library/meta/pipeline_draft.py b/cocode/pipelex_libraries/pipelines/base_library/meta/pipeline_draft.py index 70ea949..5edea69 100644 --- a/cocode/pipelex_libraries/pipelines/base_library/meta/pipeline_draft.py +++ b/cocode/pipelex_libraries/pipelines/base_library/meta/pipeline_draft.py @@ -1,6 +1,6 @@ from typing import Dict -from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint +from pipelex.core.bundles.pipelex_bundle_blueprint import PipelexBundleBlueprint as PipelexBundleBlueprintBaseModel from pipelex.core.stuffs.stuff_content import StructuredContent from pydantic import Field @@ -27,7 +27,7 @@ class PipelineDraft(StructuredContent): pipe: Dict[str, PipeDraft] = Field(default_factory=dict) -class PipelexBundleBlueprintStuff(PipelexBundleBlueprint, StructuredContent): +class PipelexBundleBlueprint(PipelexBundleBlueprintBaseModel, StructuredContent): """Complete blueprint of a pipelex bundle PLX file.""" pass diff --git a/cocode/pipelex_libraries/plugins/plugin_config.toml b/cocode/pipelex_libraries/plugins/plugin_config.toml deleted file mode 100644 index 30cde64..0000000 --- a/cocode/pipelex_libraries/plugins/plugin_config.toml +++ /dev/null @@ -1,40 +0,0 @@ - -#################################################################################################### -# Plugins config -#################################################################################################### - -[openai_config] -image_output_compression = 100 -api_key_method = "env" - -[azure_openai_config] -api_key_method = "env" - -# TODO: handle multiple azure openai accounts with different resource groups and account names for various llm model - -[perplexity_config] -api_key_method = "env" - -[xai_config] -api_key_method = "env" - -[vertexai_config] -api_key_method = "env" - -[mistral_config] -api_key_method = "env" - -[bedrock_config] -client_method = "aioboto3" - -[anthropic_config] -claude_4_reduced_tokens_limit = 8192 # use "unlimited" to enable the full 32/64K tokens Opus/Sonet but it raises streaming/timeout issues -api_key_method = "env" - -[custom_endpoint_config] -api_key_method = "env" - -[fal_config] -flux_map_quality_to_steps = { "low" = 14, "medium" = 28, "high" = 56 } -sdxl_lightning_map_quality_to_steps = { "low" = 2, "medium" = 4, "high" = 8 } - diff --git a/cocode/pipelex_libraries/templates/base_templates.toml b/cocode/pipelex_libraries/templates/base_templates.toml index fe3026d..087d362 100644 --- a/cocode/pipelex_libraries/templates/base_templates.toml +++ b/cocode/pipelex_libraries/templates/base_templates.toml @@ -1,5 +1,3 @@ - - [generic_prompts] structure_from_preliminary_text_system = "You are a data modeling expert specialized in extracting structure from text." diff --git a/pipelex.toml b/pipelex.toml index 52ef3f2..b8f8b12 100644 --- a/pipelex.toml +++ b/pipelex.toml @@ -5,16 +5,12 @@ api_key_method = "env" # "secret_provider" means that the env var are stored in your Secret Manager (See the doc for injecting a secret provider). [cogt] -[cogt.llm_config.preferred_platforms] -# This overrrides the defaults set for any llm handle -"gpt-4o-mini" = "openai" #################################################################################################### # OCR config #################################################################################################### [cogt.ocr_config] -ocr_handles = ["mistral/mistral-ocr-latest"] page_output_text_file_name = "page_text.md" [pipelex.feature_config] diff --git a/pyproject.toml b/pyproject.toml index f0e076f..217752d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,11 +9,14 @@ readme = "README.md" requires-python = ">=3.10" classifiers = [ "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Operating System :: OS Independent", ] -dependencies = ["pipelex[anthropic,google,bedrock]>=0.9.4", "PyGithub==2.4.0"] +dependencies = ["pipelex[anthropic,google,bedrock]>=0.10.2", "PyGithub==2.4.0"] [project.optional-dependencies] docs = [ diff --git a/uv.lock b/uv.lock index b8ad35c..dfd877e 100644 --- a/uv.lock +++ b/uv.lock @@ -2,7 +2,9 @@ version = 1 revision = 3 requires-python = ">=3.10" resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy'", + "python_full_version >= '3.12' and python_full_version < '3.14' and platform_python_implementation != 'PyPy'", + "python_full_version >= '3.12' and platform_python_implementation == 'PyPy'", "python_full_version == '3.11.*'", "python_full_version < '3.11'", ] @@ -344,59 +346,84 @@ wheels = [ [[package]] name = "cffi" -version = "1.17.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pycparser" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/90/07/f44ca684db4e4f08a3fdc6eeb9a0d15dc6883efc7b8c90357fdbf74e186c/cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14", size = 182191, upload-time = "2024-09-04T20:43:30.027Z" }, - { url = "https://files.pythonhosted.org/packages/08/fd/cc2fedbd887223f9f5d170c96e57cbf655df9831a6546c1727ae13fa977a/cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67", size = 178592, upload-time = "2024-09-04T20:43:32.108Z" }, - { url = "https://files.pythonhosted.org/packages/de/cc/4635c320081c78d6ffc2cab0a76025b691a91204f4aa317d568ff9280a2d/cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382", size = 426024, upload-time = "2024-09-04T20:43:34.186Z" }, - { url = "https://files.pythonhosted.org/packages/b6/7b/3b2b250f3aab91abe5f8a51ada1b717935fdaec53f790ad4100fe2ec64d1/cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702", size = 448188, upload-time = "2024-09-04T20:43:36.286Z" }, - { url = "https://files.pythonhosted.org/packages/d3/48/1b9283ebbf0ec065148d8de05d647a986c5f22586b18120020452fff8f5d/cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3", size = 455571, upload-time = "2024-09-04T20:43:38.586Z" }, - { url = "https://files.pythonhosted.org/packages/40/87/3b8452525437b40f39ca7ff70276679772ee7e8b394934ff60e63b7b090c/cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6", size = 436687, upload-time = "2024-09-04T20:43:40.084Z" }, - { url = "https://files.pythonhosted.org/packages/8d/fb/4da72871d177d63649ac449aec2e8a29efe0274035880c7af59101ca2232/cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17", size = 446211, upload-time = "2024-09-04T20:43:41.526Z" }, - { url = "https://files.pythonhosted.org/packages/ab/a0/62f00bcb411332106c02b663b26f3545a9ef136f80d5df746c05878f8c4b/cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8", size = 461325, upload-time = "2024-09-04T20:43:43.117Z" }, - { url = "https://files.pythonhosted.org/packages/36/83/76127035ed2e7e27b0787604d99da630ac3123bfb02d8e80c633f218a11d/cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e", size = 438784, upload-time = "2024-09-04T20:43:45.256Z" }, - { url = "https://files.pythonhosted.org/packages/21/81/a6cd025db2f08ac88b901b745c163d884641909641f9b826e8cb87645942/cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be", size = 461564, upload-time = "2024-09-04T20:43:46.779Z" }, - { url = "https://files.pythonhosted.org/packages/f8/fe/4d41c2f200c4a457933dbd98d3cf4e911870877bd94d9656cc0fcb390681/cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c", size = 171804, upload-time = "2024-09-04T20:43:48.186Z" }, - { url = "https://files.pythonhosted.org/packages/d1/b6/0b0f5ab93b0df4acc49cae758c81fe4e5ef26c3ae2e10cc69249dfd8b3ab/cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15", size = 181299, upload-time = "2024-09-04T20:43:49.812Z" }, - { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264, upload-time = "2024-09-04T20:43:51.124Z" }, - { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651, upload-time = "2024-09-04T20:43:52.872Z" }, - { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259, upload-time = "2024-09-04T20:43:56.123Z" }, - { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200, upload-time = "2024-09-04T20:43:57.891Z" }, - { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235, upload-time = "2024-09-04T20:44:00.18Z" }, - { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721, upload-time = "2024-09-04T20:44:01.585Z" }, - { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242, upload-time = "2024-09-04T20:44:03.467Z" }, - { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999, upload-time = "2024-09-04T20:44:05.023Z" }, - { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242, upload-time = "2024-09-04T20:44:06.444Z" }, - { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604, upload-time = "2024-09-04T20:44:08.206Z" }, - { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727, upload-time = "2024-09-04T20:44:09.481Z" }, - { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400, upload-time = "2024-09-04T20:44:10.873Z" }, - { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178, upload-time = "2024-09-04T20:44:12.232Z" }, - { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840, upload-time = "2024-09-04T20:44:13.739Z" }, - { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803, upload-time = "2024-09-04T20:44:15.231Z" }, - { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850, upload-time = "2024-09-04T20:44:17.188Z" }, - { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729, upload-time = "2024-09-04T20:44:18.688Z" }, - { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256, upload-time = "2024-09-04T20:44:20.248Z" }, - { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424, upload-time = "2024-09-04T20:44:21.673Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568, upload-time = "2024-09-04T20:44:23.245Z" }, - { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736, upload-time = "2024-09-04T20:44:24.757Z" }, - { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448, upload-time = "2024-09-04T20:44:26.208Z" }, - { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976, upload-time = "2024-09-04T20:44:27.578Z" }, - { url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989, upload-time = "2024-09-04T20:44:28.956Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802, upload-time = "2024-09-04T20:44:30.289Z" }, - { url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792, upload-time = "2024-09-04T20:44:32.01Z" }, - { url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893, upload-time = "2024-09-04T20:44:33.606Z" }, - { url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810, upload-time = "2024-09-04T20:44:35.191Z" }, - { url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200, upload-time = "2024-09-04T20:44:36.743Z" }, - { url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447, upload-time = "2024-09-04T20:44:38.492Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358, upload-time = "2024-09-04T20:44:40.046Z" }, - { url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469, upload-time = "2024-09-04T20:44:41.616Z" }, - { url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475, upload-time = "2024-09-04T20:44:43.733Z" }, - { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009, upload-time = "2024-09-04T20:44:45.309Z" }, +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "(python_full_version < '3.12' and implementation_name != 'PyPy') or (implementation_name != 'PyPy' and platform_python_implementation != 'PyPy')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/d7/516d984057745a6cd96575eea814fe1edd6646ee6efd552fb7b0921dec83/cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44", size = 184283, upload-time = "2025-09-08T23:22:08.01Z" }, + { url = "https://files.pythonhosted.org/packages/9e/84/ad6a0b408daa859246f57c03efd28e5dd1b33c21737c2db84cae8c237aa5/cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49", size = 180504, upload-time = "2025-09-08T23:22:10.637Z" }, + { url = "https://files.pythonhosted.org/packages/50/bd/b1a6362b80628111e6653c961f987faa55262b4002fcec42308cad1db680/cffi-2.0.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c", size = 208811, upload-time = "2025-09-08T23:22:12.267Z" }, + { url = "https://files.pythonhosted.org/packages/4f/27/6933a8b2562d7bd1fb595074cf99cc81fc3789f6a6c05cdabb46284a3188/cffi-2.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb", size = 216402, upload-time = "2025-09-08T23:22:13.455Z" }, + { url = "https://files.pythonhosted.org/packages/05/eb/b86f2a2645b62adcfff53b0dd97e8dfafb5c8aa864bd0d9a2c2049a0d551/cffi-2.0.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0", size = 203217, upload-time = "2025-09-08T23:22:14.596Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e0/6cbe77a53acf5acc7c08cc186c9928864bd7c005f9efd0d126884858a5fe/cffi-2.0.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4", size = 203079, upload-time = "2025-09-08T23:22:15.769Z" }, + { url = "https://files.pythonhosted.org/packages/98/29/9b366e70e243eb3d14a5cb488dfd3a0b6b2f1fb001a203f653b93ccfac88/cffi-2.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453", size = 216475, upload-time = "2025-09-08T23:22:17.427Z" }, + { url = "https://files.pythonhosted.org/packages/21/7a/13b24e70d2f90a322f2900c5d8e1f14fa7e2a6b3332b7309ba7b2ba51a5a/cffi-2.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495", size = 218829, upload-time = "2025-09-08T23:22:19.069Z" }, + { url = "https://files.pythonhosted.org/packages/60/99/c9dc110974c59cc981b1f5b66e1d8af8af764e00f0293266824d9c4254bc/cffi-2.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5", size = 211211, upload-time = "2025-09-08T23:22:20.588Z" }, + { url = "https://files.pythonhosted.org/packages/49/72/ff2d12dbf21aca1b32a40ed792ee6b40f6dc3a9cf1644bd7ef6e95e0ac5e/cffi-2.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb", size = 218036, upload-time = "2025-09-08T23:22:22.143Z" }, + { url = "https://files.pythonhosted.org/packages/e2/cc/027d7fb82e58c48ea717149b03bcadcbdc293553edb283af792bd4bcbb3f/cffi-2.0.0-cp310-cp310-win32.whl", hash = "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a", size = 172184, upload-time = "2025-09-08T23:22:23.328Z" }, + { url = "https://files.pythonhosted.org/packages/33/fa/072dd15ae27fbb4e06b437eb6e944e75b068deb09e2a2826039e49ee2045/cffi-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739", size = 182790, upload-time = "2025-09-08T23:22:24.752Z" }, + { url = "https://files.pythonhosted.org/packages/12/4a/3dfd5f7850cbf0d06dc84ba9aa00db766b52ca38d8b86e3a38314d52498c/cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", size = 184344, upload-time = "2025-09-08T23:22:26.456Z" }, + { url = "https://files.pythonhosted.org/packages/4f/8b/f0e4c441227ba756aafbe78f117485b25bb26b1c059d01f137fa6d14896b/cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", size = 180560, upload-time = "2025-09-08T23:22:28.197Z" }, + { url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613, upload-time = "2025-09-08T23:22:29.475Z" }, + { url = "https://files.pythonhosted.org/packages/b8/56/6033f5e86e8cc9bb629f0077ba71679508bdf54a9a5e112a3c0b91870332/cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", size = 216476, upload-time = "2025-09-08T23:22:31.063Z" }, + { url = "https://files.pythonhosted.org/packages/dc/7f/55fecd70f7ece178db2f26128ec41430d8720f2d12ca97bf8f0a628207d5/cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", size = 203374, upload-time = "2025-09-08T23:22:32.507Z" }, + { url = "https://files.pythonhosted.org/packages/84/ef/a7b77c8bdc0f77adc3b46888f1ad54be8f3b7821697a7b89126e829e676a/cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", size = 202597, upload-time = "2025-09-08T23:22:34.132Z" }, + { url = "https://files.pythonhosted.org/packages/d7/91/500d892b2bf36529a75b77958edfcd5ad8e2ce4064ce2ecfeab2125d72d1/cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", size = 215574, upload-time = "2025-09-08T23:22:35.443Z" }, + { url = "https://files.pythonhosted.org/packages/44/64/58f6255b62b101093d5df22dcb752596066c7e89dd725e0afaed242a61be/cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", size = 218971, upload-time = "2025-09-08T23:22:36.805Z" }, + { url = "https://files.pythonhosted.org/packages/ab/49/fa72cebe2fd8a55fbe14956f9970fe8eb1ac59e5df042f603ef7c8ba0adc/cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", size = 211972, upload-time = "2025-09-08T23:22:38.436Z" }, + { url = "https://files.pythonhosted.org/packages/0b/28/dd0967a76aab36731b6ebfe64dec4e981aff7e0608f60c2d46b46982607d/cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", size = 217078, upload-time = "2025-09-08T23:22:39.776Z" }, + { url = "https://files.pythonhosted.org/packages/2b/c0/015b25184413d7ab0a410775fdb4a50fca20f5589b5dab1dbbfa3baad8ce/cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", size = 172076, upload-time = "2025-09-08T23:22:40.95Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8f/dc5531155e7070361eb1b7e4c1a9d896d0cb21c49f807a6c03fd63fc877e/cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", size = 182820, upload-time = "2025-09-08T23:22:42.463Z" }, + { url = "https://files.pythonhosted.org/packages/95/5c/1b493356429f9aecfd56bc171285a4c4ac8697f76e9bbbbb105e537853a1/cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", size = 177635, upload-time = "2025-09-08T23:22:43.623Z" }, + { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, + { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, + { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, + { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, + { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, + { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, + { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, + { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, + { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" }, + { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" }, + { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" }, + { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" }, + { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" }, + { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" }, + { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" }, + { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320, upload-time = "2025-09-08T23:23:18.087Z" }, + { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487, upload-time = "2025-09-08T23:23:19.622Z" }, + { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" }, + { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" }, + { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" }, + { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" }, + { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" }, + { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" }, + { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" }, + { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773, upload-time = "2025-09-08T23:23:29.347Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013, upload-time = "2025-09-08T23:23:30.63Z" }, + { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" }, + { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" }, + { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" }, + { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" }, + { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" }, + { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" }, + { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, ] [[package]] @@ -526,7 +553,7 @@ requires-dist = [ { name = "mkdocs-material", marker = "extra == 'docs'", specifier = "==9.6.14" }, { name = "mkdocs-meta-manager", marker = "extra == 'docs'", specifier = "==1.1.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.11.2" }, - { name = "pipelex", extras = ["anthropic", "google", "bedrock"], specifier = ">=0.9.4" }, + { name = "pipelex", extras = ["anthropic", "google", "bedrock"], specifier = ">=0.10.2" }, { name = "pygithub", specifier = "==2.4.0" }, { name = "pyright", marker = "extra == 'dev'", specifier = "==1.1.398" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.3" }, @@ -627,49 +654,67 @@ toml = [ [[package]] name = "cryptography" -version = "45.0.5" +version = "46.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/95/1e/49527ac611af559665f71cbb8f92b332b5ec9c6fbc4e88b0f8e92f5e85df/cryptography-45.0.5.tar.gz", hash = "sha256:72e76caa004ab63accdf26023fccd1d087f6d90ec6048ff33ad0445abf7f605a", size = 744903, upload-time = "2025-07-02T13:06:25.941Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/fb/09e28bc0c46d2c547085e60897fea96310574c70fb21cd58a730a45f3403/cryptography-45.0.5-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:101ee65078f6dd3e5a028d4f19c07ffa4dd22cce6a20eaa160f8b5219911e7d8", size = 7043092, upload-time = "2025-07-02T13:05:01.514Z" }, - { url = "https://files.pythonhosted.org/packages/b1/05/2194432935e29b91fb649f6149c1a4f9e6d3d9fc880919f4ad1bcc22641e/cryptography-45.0.5-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3a264aae5f7fbb089dbc01e0242d3b67dffe3e6292e1f5182122bdf58e65215d", size = 4205926, upload-time = "2025-07-02T13:05:04.741Z" }, - { url = "https://files.pythonhosted.org/packages/07/8b/9ef5da82350175e32de245646b1884fc01124f53eb31164c77f95a08d682/cryptography-45.0.5-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e74d30ec9c7cb2f404af331d5b4099a9b322a8a6b25c4632755c8757345baac5", size = 4429235, upload-time = "2025-07-02T13:05:07.084Z" }, - { url = "https://files.pythonhosted.org/packages/7c/e1/c809f398adde1994ee53438912192d92a1d0fc0f2d7582659d9ef4c28b0c/cryptography-45.0.5-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3af26738f2db354aafe492fb3869e955b12b2ef2e16908c8b9cb928128d42c57", size = 4209785, upload-time = "2025-07-02T13:05:09.321Z" }, - { url = "https://files.pythonhosted.org/packages/d0/8b/07eb6bd5acff58406c5e806eff34a124936f41a4fb52909ffa4d00815f8c/cryptography-45.0.5-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e6c00130ed423201c5bc5544c23359141660b07999ad82e34e7bb8f882bb78e0", size = 3893050, upload-time = "2025-07-02T13:05:11.069Z" }, - { url = "https://files.pythonhosted.org/packages/ec/ef/3333295ed58d900a13c92806b67e62f27876845a9a908c939f040887cca9/cryptography-45.0.5-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:dd420e577921c8c2d31289536c386aaa30140b473835e97f83bc71ea9d2baf2d", size = 4457379, upload-time = "2025-07-02T13:05:13.32Z" }, - { url = "https://files.pythonhosted.org/packages/d9/9d/44080674dee514dbb82b21d6fa5d1055368f208304e2ab1828d85c9de8f4/cryptography-45.0.5-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:d05a38884db2ba215218745f0781775806bde4f32e07b135348355fe8e4991d9", size = 4209355, upload-time = "2025-07-02T13:05:15.017Z" }, - { url = "https://files.pythonhosted.org/packages/c9/d8/0749f7d39f53f8258e5c18a93131919ac465ee1f9dccaf1b3f420235e0b5/cryptography-45.0.5-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:ad0caded895a00261a5b4aa9af828baede54638754b51955a0ac75576b831b27", size = 4456087, upload-time = "2025-07-02T13:05:16.945Z" }, - { url = "https://files.pythonhosted.org/packages/09/d7/92acac187387bf08902b0bf0699816f08553927bdd6ba3654da0010289b4/cryptography-45.0.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9024beb59aca9d31d36fcdc1604dd9bbeed0a55bface9f1908df19178e2f116e", size = 4332873, upload-time = "2025-07-02T13:05:18.743Z" }, - { url = "https://files.pythonhosted.org/packages/03/c2/840e0710da5106a7c3d4153c7215b2736151bba60bf4491bdb421df5056d/cryptography-45.0.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:91098f02ca81579c85f66df8a588c78f331ca19089763d733e34ad359f474174", size = 4564651, upload-time = "2025-07-02T13:05:21.382Z" }, - { url = "https://files.pythonhosted.org/packages/2e/92/cc723dd6d71e9747a887b94eb3827825c6c24b9e6ce2bb33b847d31d5eaa/cryptography-45.0.5-cp311-abi3-win32.whl", hash = "sha256:926c3ea71a6043921050eaa639137e13dbe7b4ab25800932a8498364fc1abec9", size = 2929050, upload-time = "2025-07-02T13:05:23.39Z" }, - { url = "https://files.pythonhosted.org/packages/1f/10/197da38a5911a48dd5389c043de4aec4b3c94cb836299b01253940788d78/cryptography-45.0.5-cp311-abi3-win_amd64.whl", hash = "sha256:b85980d1e345fe769cfc57c57db2b59cff5464ee0c045d52c0df087e926fbe63", size = 3403224, upload-time = "2025-07-02T13:05:25.202Z" }, - { url = "https://files.pythonhosted.org/packages/fe/2b/160ce8c2765e7a481ce57d55eba1546148583e7b6f85514472b1d151711d/cryptography-45.0.5-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:f3562c2f23c612f2e4a6964a61d942f891d29ee320edb62ff48ffb99f3de9ae8", size = 7017143, upload-time = "2025-07-02T13:05:27.229Z" }, - { url = "https://files.pythonhosted.org/packages/c2/e7/2187be2f871c0221a81f55ee3105d3cf3e273c0a0853651d7011eada0d7e/cryptography-45.0.5-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3fcfbefc4a7f332dece7272a88e410f611e79458fab97b5efe14e54fe476f4fd", size = 4197780, upload-time = "2025-07-02T13:05:29.299Z" }, - { url = "https://files.pythonhosted.org/packages/b9/cf/84210c447c06104e6be9122661159ad4ce7a8190011669afceeaea150524/cryptography-45.0.5-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:460f8c39ba66af7db0545a8c6f2eabcbc5a5528fc1cf6c3fa9a1e44cec33385e", size = 4420091, upload-time = "2025-07-02T13:05:31.221Z" }, - { url = "https://files.pythonhosted.org/packages/3e/6a/cb8b5c8bb82fafffa23aeff8d3a39822593cee6e2f16c5ca5c2ecca344f7/cryptography-45.0.5-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9b4cf6318915dccfe218e69bbec417fdd7c7185aa7aab139a2c0beb7468c89f0", size = 4198711, upload-time = "2025-07-02T13:05:33.062Z" }, - { url = "https://files.pythonhosted.org/packages/04/f7/36d2d69df69c94cbb2473871926daf0f01ad8e00fe3986ac3c1e8c4ca4b3/cryptography-45.0.5-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2089cc8f70a6e454601525e5bf2779e665d7865af002a5dec8d14e561002e135", size = 3883299, upload-time = "2025-07-02T13:05:34.94Z" }, - { url = "https://files.pythonhosted.org/packages/82/c7/f0ea40f016de72f81288e9fe8d1f6748036cb5ba6118774317a3ffc6022d/cryptography-45.0.5-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0027d566d65a38497bc37e0dd7c2f8ceda73597d2ac9ba93810204f56f52ebc7", size = 4450558, upload-time = "2025-07-02T13:05:37.288Z" }, - { url = "https://files.pythonhosted.org/packages/06/ae/94b504dc1a3cdf642d710407c62e86296f7da9e66f27ab12a1ee6fdf005b/cryptography-45.0.5-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:be97d3a19c16a9be00edf79dca949c8fa7eff621763666a145f9f9535a5d7f42", size = 4198020, upload-time = "2025-07-02T13:05:39.102Z" }, - { url = "https://files.pythonhosted.org/packages/05/2b/aaf0adb845d5dabb43480f18f7ca72e94f92c280aa983ddbd0bcd6ecd037/cryptography-45.0.5-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:7760c1c2e1a7084153a0f68fab76e754083b126a47d0117c9ed15e69e2103492", size = 4449759, upload-time = "2025-07-02T13:05:41.398Z" }, - { url = "https://files.pythonhosted.org/packages/91/e4/f17e02066de63e0100a3a01b56f8f1016973a1d67551beaf585157a86b3f/cryptography-45.0.5-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6ff8728d8d890b3dda5765276d1bc6fb099252915a2cd3aff960c4c195745dd0", size = 4319991, upload-time = "2025-07-02T13:05:43.64Z" }, - { url = "https://files.pythonhosted.org/packages/f2/2e/e2dbd629481b499b14516eed933f3276eb3239f7cee2dcfa4ee6b44d4711/cryptography-45.0.5-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7259038202a47fdecee7e62e0fd0b0738b6daa335354396c6ddebdbe1206af2a", size = 4554189, upload-time = "2025-07-02T13:05:46.045Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ea/a78a0c38f4c8736287b71c2ea3799d173d5ce778c7d6e3c163a95a05ad2a/cryptography-45.0.5-cp37-abi3-win32.whl", hash = "sha256:1e1da5accc0c750056c556a93c3e9cb828970206c68867712ca5805e46dc806f", size = 2911769, upload-time = "2025-07-02T13:05:48.329Z" }, - { url = "https://files.pythonhosted.org/packages/79/b3/28ac139109d9005ad3f6b6f8976ffede6706a6478e21c889ce36c840918e/cryptography-45.0.5-cp37-abi3-win_amd64.whl", hash = "sha256:90cb0a7bb35959f37e23303b7eed0a32280510030daba3f7fdfbb65defde6a97", size = 3390016, upload-time = "2025-07-02T13:05:50.811Z" }, - { url = "https://files.pythonhosted.org/packages/f8/8b/34394337abe4566848a2bd49b26bcd4b07fd466afd3e8cce4cb79a390869/cryptography-45.0.5-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:206210d03c1193f4e1ff681d22885181d47efa1ab3018766a7b32a7b3d6e6afd", size = 3575762, upload-time = "2025-07-02T13:05:53.166Z" }, - { url = "https://files.pythonhosted.org/packages/8b/5d/a19441c1e89afb0f173ac13178606ca6fab0d3bd3ebc29e9ed1318b507fc/cryptography-45.0.5-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c648025b6840fe62e57107e0a25f604db740e728bd67da4f6f060f03017d5097", size = 4140906, upload-time = "2025-07-02T13:05:55.914Z" }, - { url = "https://files.pythonhosted.org/packages/4b/db/daceb259982a3c2da4e619f45b5bfdec0e922a23de213b2636e78ef0919b/cryptography-45.0.5-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b8fa8b0a35a9982a3c60ec79905ba5bb090fc0b9addcfd3dc2dd04267e45f25e", size = 4374411, upload-time = "2025-07-02T13:05:57.814Z" }, - { url = "https://files.pythonhosted.org/packages/6a/35/5d06ad06402fc522c8bf7eab73422d05e789b4e38fe3206a85e3d6966c11/cryptography-45.0.5-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:14d96584701a887763384f3c47f0ca7c1cce322aa1c31172680eb596b890ec30", size = 4140942, upload-time = "2025-07-02T13:06:00.137Z" }, - { url = "https://files.pythonhosted.org/packages/65/79/020a5413347e44c382ef1f7f7e7a66817cd6273e3e6b5a72d18177b08b2f/cryptography-45.0.5-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:57c816dfbd1659a367831baca4b775b2a5b43c003daf52e9d57e1d30bc2e1b0e", size = 4374079, upload-time = "2025-07-02T13:06:02.043Z" }, - { url = "https://files.pythonhosted.org/packages/9b/c5/c0e07d84a9a2a8a0ed4f865e58f37c71af3eab7d5e094ff1b21f3f3af3bc/cryptography-45.0.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b9e38e0a83cd51e07f5a48ff9691cae95a79bea28fe4ded168a8e5c6c77e819d", size = 3321362, upload-time = "2025-07-02T13:06:04.463Z" }, - { url = "https://files.pythonhosted.org/packages/c0/71/9bdbcfd58d6ff5084687fe722c58ac718ebedbc98b9f8f93781354e6d286/cryptography-45.0.5-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8c4a6ff8a30e9e3d38ac0539e9a9e02540ab3f827a3394f8852432f6b0ea152e", size = 3587878, upload-time = "2025-07-02T13:06:06.339Z" }, - { url = "https://files.pythonhosted.org/packages/f0/63/83516cfb87f4a8756eaa4203f93b283fda23d210fc14e1e594bd5f20edb6/cryptography-45.0.5-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:bd4c45986472694e5121084c6ebbd112aa919a25e783b87eb95953c9573906d6", size = 4152447, upload-time = "2025-07-02T13:06:08.345Z" }, - { url = "https://files.pythonhosted.org/packages/22/11/d2823d2a5a0bd5802b3565437add16f5c8ce1f0778bf3822f89ad2740a38/cryptography-45.0.5-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:982518cd64c54fcada9d7e5cf28eabd3ee76bd03ab18e08a48cad7e8b6f31b18", size = 4386778, upload-time = "2025-07-02T13:06:10.263Z" }, - { url = "https://files.pythonhosted.org/packages/5f/38/6bf177ca6bce4fe14704ab3e93627c5b0ca05242261a2e43ef3168472540/cryptography-45.0.5-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:12e55281d993a793b0e883066f590c1ae1e802e3acb67f8b442e721e475e6463", size = 4151627, upload-time = "2025-07-02T13:06:13.097Z" }, - { url = "https://files.pythonhosted.org/packages/38/6a/69fc67e5266bff68a91bcb81dff8fb0aba4d79a78521a08812048913e16f/cryptography-45.0.5-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:5aa1e32983d4443e310f726ee4b071ab7569f58eedfdd65e9675484a4eb67bd1", size = 4385593, upload-time = "2025-07-02T13:06:15.689Z" }, - { url = "https://files.pythonhosted.org/packages/f6/34/31a1604c9a9ade0fdab61eb48570e09a796f4d9836121266447b0eaf7feb/cryptography-45.0.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:e357286c1b76403dd384d938f93c46b2b058ed4dfcdce64a770f0537ed3feb6f", size = 3331106, upload-time = "2025-07-02T13:06:18.058Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/a9/62/e3664e6ffd7743e1694b244dde70b43a394f6f7fbcacf7014a8ff5197c73/cryptography-46.0.1.tar.gz", hash = "sha256:ed570874e88f213437f5cf758f9ef26cbfc3f336d889b1e592ee11283bb8d1c7", size = 749198, upload-time = "2025-09-17T00:10:35.797Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/8c/44ee01267ec01e26e43ebfdae3f120ec2312aa72fa4c0507ebe41a26739f/cryptography-46.0.1-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:1cd6d50c1a8b79af1a6f703709d8973845f677c8e97b1268f5ff323d38ce8475", size = 7285044, upload-time = "2025-09-17T00:08:36.807Z" }, + { url = "https://files.pythonhosted.org/packages/22/59/9ae689a25047e0601adfcb159ec4f83c0b4149fdb5c3030cc94cd218141d/cryptography-46.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0ff483716be32690c14636e54a1f6e2e1b7bf8e22ca50b989f88fa1b2d287080", size = 4308182, upload-time = "2025-09-17T00:08:39.388Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ee/ca6cc9df7118f2fcd142c76b1da0f14340d77518c05b1ebfbbabca6b9e7d/cryptography-46.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9873bf7c1f2a6330bdfe8621e7ce64b725784f9f0c3a6a55c3047af5849f920e", size = 4572393, upload-time = "2025-09-17T00:08:41.663Z" }, + { url = "https://files.pythonhosted.org/packages/7f/a3/0f5296f63815d8e985922b05c31f77ce44787b3127a67c0b7f70f115c45f/cryptography-46.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0dfb7c88d4462a0cfdd0d87a3c245a7bc3feb59de101f6ff88194f740f72eda6", size = 4308400, upload-time = "2025-09-17T00:08:43.559Z" }, + { url = "https://files.pythonhosted.org/packages/5d/8c/74fcda3e4e01be1d32775d5b4dd841acaac3c1b8fa4d0774c7ac8d52463d/cryptography-46.0.1-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e22801b61613ebdebf7deb18b507919e107547a1d39a3b57f5f855032dd7cfb8", size = 4015786, upload-time = "2025-09-17T00:08:45.758Z" }, + { url = "https://files.pythonhosted.org/packages/dc/b8/85d23287baeef273b0834481a3dd55bbed3a53587e3b8d9f0898235b8f91/cryptography-46.0.1-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:757af4f6341ce7a1e47c326ca2a81f41d236070217e5fbbad61bbfe299d55d28", size = 4982606, upload-time = "2025-09-17T00:08:47.602Z" }, + { url = "https://files.pythonhosted.org/packages/e5/d3/de61ad5b52433b389afca0bc70f02a7a1f074651221f599ce368da0fe437/cryptography-46.0.1-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f7a24ea78de345cfa7f6a8d3bde8b242c7fac27f2bd78fa23474ca38dfaeeab9", size = 4604234, upload-time = "2025-09-17T00:08:49.879Z" }, + { url = "https://files.pythonhosted.org/packages/dc/1f/dbd4d6570d84748439237a7478d124ee0134bf166ad129267b7ed8ea6d22/cryptography-46.0.1-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:9e8776dac9e660c22241b6587fae51a67b4b0147daa4d176b172c3ff768ad736", size = 4307669, upload-time = "2025-09-17T00:08:52.321Z" }, + { url = "https://files.pythonhosted.org/packages/ec/fd/ca0a14ce7f0bfe92fa727aacaf2217eb25eb7e4ed513b14d8e03b26e63ed/cryptography-46.0.1-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9f40642a140c0c8649987027867242b801486865277cbabc8c6059ddef16dc8b", size = 4947579, upload-time = "2025-09-17T00:08:54.697Z" }, + { url = "https://files.pythonhosted.org/packages/89/6b/09c30543bb93401f6f88fce556b3bdbb21e55ae14912c04b7bf355f5f96c/cryptography-46.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:449ef2b321bec7d97ef2c944173275ebdab78f3abdd005400cc409e27cd159ab", size = 4603669, upload-time = "2025-09-17T00:08:57.16Z" }, + { url = "https://files.pythonhosted.org/packages/23/9a/38cb01cb09ce0adceda9fc627c9cf98eb890fc8d50cacbe79b011df20f8a/cryptography-46.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2dd339ba3345b908fa3141ddba4025568fa6fd398eabce3ef72a29ac2d73ad75", size = 4435828, upload-time = "2025-09-17T00:08:59.606Z" }, + { url = "https://files.pythonhosted.org/packages/0f/53/435b5c36a78d06ae0bef96d666209b0ecd8f8181bfe4dda46536705df59e/cryptography-46.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7411c910fb2a412053cf33cfad0153ee20d27e256c6c3f14d7d7d1d9fec59fd5", size = 4709553, upload-time = "2025-09-17T00:09:01.832Z" }, + { url = "https://files.pythonhosted.org/packages/f5/c4/0da6e55595d9b9cd3b6eb5dc22f3a07ded7f116a3ea72629cab595abb804/cryptography-46.0.1-cp311-abi3-win32.whl", hash = "sha256:cbb8e769d4cac884bb28e3ff620ef1001b75588a5c83c9c9f1fdc9afbe7f29b0", size = 3058327, upload-time = "2025-09-17T00:09:03.726Z" }, + { url = "https://files.pythonhosted.org/packages/95/0f/cd29a35e0d6e78a0ee61793564c8cff0929c38391cb0de27627bdc7525aa/cryptography-46.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:92e8cfe8bd7dd86eac0a677499894862cd5cc2fd74de917daa881d00871ac8e7", size = 3523893, upload-time = "2025-09-17T00:09:06.272Z" }, + { url = "https://files.pythonhosted.org/packages/f2/dd/eea390f3e78432bc3d2f53952375f8b37cb4d37783e626faa6a51e751719/cryptography-46.0.1-cp311-abi3-win_arm64.whl", hash = "sha256:db5597a4c7353b2e5fb05a8e6cb74b56a4658a2b7bf3cb6b1821ae7e7fd6eaa0", size = 2932145, upload-time = "2025-09-17T00:09:08.568Z" }, + { url = "https://files.pythonhosted.org/packages/0a/fb/c73588561afcd5e24b089952bd210b14676c0c5bf1213376350ae111945c/cryptography-46.0.1-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:4c49eda9a23019e11d32a0eb51a27b3e7ddedde91e099c0ac6373e3aacc0d2ee", size = 7193928, upload-time = "2025-09-17T00:09:10.595Z" }, + { url = "https://files.pythonhosted.org/packages/26/34/0ff0bb2d2c79f25a2a63109f3b76b9108a906dd2a2eb5c1d460b9938adbb/cryptography-46.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9babb7818fdd71394e576cf26c5452df77a355eac1a27ddfa24096665a27f8fd", size = 4293515, upload-time = "2025-09-17T00:09:12.861Z" }, + { url = "https://files.pythonhosted.org/packages/df/b7/d4f848aee24ecd1be01db6c42c4a270069a4f02a105d9c57e143daf6cf0f/cryptography-46.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9f2c4cc63be3ef43c0221861177cee5d14b505cd4d4599a89e2cd273c4d3542a", size = 4545619, upload-time = "2025-09-17T00:09:15.397Z" }, + { url = "https://files.pythonhosted.org/packages/44/a5/42fedefc754fd1901e2d95a69815ea4ec8a9eed31f4c4361fcab80288661/cryptography-46.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:41c281a74df173876da1dc9a9b6953d387f06e3d3ed9284e3baae3ab3f40883a", size = 4299160, upload-time = "2025-09-17T00:09:17.155Z" }, + { url = "https://files.pythonhosted.org/packages/86/a1/cd21174f56e769c831fbbd6399a1b7519b0ff6280acec1b826d7b072640c/cryptography-46.0.1-cp314-cp314t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0a17377fa52563d730248ba1f68185461fff36e8bc75d8787a7dd2e20a802b7a", size = 3994491, upload-time = "2025-09-17T00:09:18.971Z" }, + { url = "https://files.pythonhosted.org/packages/8d/2f/a8cbfa1c029987ddc746fd966711d4fa71efc891d37fbe9f030fe5ab4eec/cryptography-46.0.1-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:0d1922d9280e08cde90b518a10cd66831f632960a8d08cb3418922d83fce6f12", size = 4960157, upload-time = "2025-09-17T00:09:20.923Z" }, + { url = "https://files.pythonhosted.org/packages/67/ae/63a84e6789e0d5a2502edf06b552bcb0fa9ff16147265d5c44a211942abe/cryptography-46.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:af84e8e99f1a82cea149e253014ea9dc89f75b82c87bb6c7242203186f465129", size = 4577263, upload-time = "2025-09-17T00:09:23.356Z" }, + { url = "https://files.pythonhosted.org/packages/ef/8f/1b9fa8e92bd9cbcb3b7e1e593a5232f2c1e6f9bd72b919c1a6b37d315f92/cryptography-46.0.1-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ef648d2c690703501714588b2ba640facd50fd16548133b11b2859e8655a69da", size = 4298703, upload-time = "2025-09-17T00:09:25.566Z" }, + { url = "https://files.pythonhosted.org/packages/c3/af/bb95db070e73fea3fae31d8a69ac1463d89d1c084220f549b00dd01094a8/cryptography-46.0.1-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:e94eb5fa32a8a9f9bf991f424f002913e3dd7c699ef552db9b14ba6a76a6313b", size = 4926363, upload-time = "2025-09-17T00:09:27.451Z" }, + { url = "https://files.pythonhosted.org/packages/f5/3b/d8fb17ffeb3a83157a1cc0aa5c60691d062aceecba09c2e5e77ebfc1870c/cryptography-46.0.1-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:534b96c0831855e29fc3b069b085fd185aa5353033631a585d5cd4dd5d40d657", size = 4576958, upload-time = "2025-09-17T00:09:29.924Z" }, + { url = "https://files.pythonhosted.org/packages/d9/46/86bc3a05c10c8aa88c8ae7e953a8b4e407c57823ed201dbcba55c4d655f4/cryptography-46.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f9b55038b5c6c47559aa33626d8ecd092f354e23de3c6975e4bb205df128a2a0", size = 4422507, upload-time = "2025-09-17T00:09:32.222Z" }, + { url = "https://files.pythonhosted.org/packages/a8/4e/387e5a21dfd2b4198e74968a541cfd6128f66f8ec94ed971776e15091ac3/cryptography-46.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ec13b7105117dbc9afd023300fb9954d72ca855c274fe563e72428ece10191c0", size = 4683964, upload-time = "2025-09-17T00:09:34.118Z" }, + { url = "https://files.pythonhosted.org/packages/25/a3/f9f5907b166adb8f26762071474b38bbfcf89858a5282f032899075a38a1/cryptography-46.0.1-cp314-cp314t-win32.whl", hash = "sha256:504e464944f2c003a0785b81668fe23c06f3b037e9cb9f68a7c672246319f277", size = 3029705, upload-time = "2025-09-17T00:09:36.381Z" }, + { url = "https://files.pythonhosted.org/packages/12/66/4d3a4f1850db2e71c2b1628d14b70b5e4c1684a1bd462f7fffb93c041c38/cryptography-46.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c52fded6383f7e20eaf70a60aeddd796b3677c3ad2922c801be330db62778e05", size = 3502175, upload-time = "2025-09-17T00:09:38.261Z" }, + { url = "https://files.pythonhosted.org/packages/52/c7/9f10ad91435ef7d0d99a0b93c4360bea3df18050ff5b9038c489c31ac2f5/cryptography-46.0.1-cp314-cp314t-win_arm64.whl", hash = "sha256:9495d78f52c804b5ec8878b5b8c7873aa8e63db9cd9ee387ff2db3fffe4df784", size = 2912354, upload-time = "2025-09-17T00:09:40.078Z" }, + { url = "https://files.pythonhosted.org/packages/98/e5/fbd632385542a3311915976f88e0dfcf09e62a3fc0aff86fb6762162a24d/cryptography-46.0.1-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:d84c40bdb8674c29fa192373498b6cb1e84f882889d21a471b45d1f868d8d44b", size = 7255677, upload-time = "2025-09-17T00:09:42.407Z" }, + { url = "https://files.pythonhosted.org/packages/56/3e/13ce6eab9ad6eba1b15a7bd476f005a4c1b3f299f4c2f32b22408b0edccf/cryptography-46.0.1-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9ed64e5083fa806709e74fc5ea067dfef9090e5b7a2320a49be3c9df3583a2d8", size = 4301110, upload-time = "2025-09-17T00:09:45.614Z" }, + { url = "https://files.pythonhosted.org/packages/a2/67/65dc233c1ddd688073cf7b136b06ff4b84bf517ba5529607c9d79720fc67/cryptography-46.0.1-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:341fb7a26bc9d6093c1b124b9f13acc283d2d51da440b98b55ab3f79f2522ead", size = 4562369, upload-time = "2025-09-17T00:09:47.601Z" }, + { url = "https://files.pythonhosted.org/packages/17/db/d64ae4c6f4e98c3dac5bf35dd4d103f4c7c345703e43560113e5e8e31b2b/cryptography-46.0.1-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6ef1488967e729948d424d09c94753d0167ce59afba8d0f6c07a22b629c557b2", size = 4302126, upload-time = "2025-09-17T00:09:49.335Z" }, + { url = "https://files.pythonhosted.org/packages/3d/19/5f1eea17d4805ebdc2e685b7b02800c4f63f3dd46cfa8d4c18373fea46c8/cryptography-46.0.1-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7823bc7cdf0b747ecfb096d004cc41573c2f5c7e3a29861603a2871b43d3ef32", size = 4009431, upload-time = "2025-09-17T00:09:51.239Z" }, + { url = "https://files.pythonhosted.org/packages/81/b5/229ba6088fe7abccbfe4c5edb96c7a5ad547fac5fdd0d40aa6ea540b2985/cryptography-46.0.1-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:f736ab8036796f5a119ff8211deda416f8c15ce03776db704a7a4e17381cb2ef", size = 4980739, upload-time = "2025-09-17T00:09:54.181Z" }, + { url = "https://files.pythonhosted.org/packages/3a/9c/50aa38907b201e74bc43c572f9603fa82b58e831bd13c245613a23cff736/cryptography-46.0.1-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:e46710a240a41d594953012213ea8ca398cd2448fbc5d0f1be8160b5511104a0", size = 4592289, upload-time = "2025-09-17T00:09:56.731Z" }, + { url = "https://files.pythonhosted.org/packages/5a/33/229858f8a5bb22f82468bb285e9f4c44a31978d5f5830bb4ea1cf8a4e454/cryptography-46.0.1-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:84ef1f145de5aee82ea2447224dc23f065ff4cc5791bb3b506615957a6ba8128", size = 4301815, upload-time = "2025-09-17T00:09:58.548Z" }, + { url = "https://files.pythonhosted.org/packages/52/cb/b76b2c87fbd6ed4a231884bea3ce073406ba8e2dae9defad910d33cbf408/cryptography-46.0.1-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9394c7d5a7565ac5f7d9ba38b2617448eba384d7b107b262d63890079fad77ca", size = 4943251, upload-time = "2025-09-17T00:10:00.475Z" }, + { url = "https://files.pythonhosted.org/packages/94/0f/f66125ecf88e4cb5b8017ff43f3a87ede2d064cb54a1c5893f9da9d65093/cryptography-46.0.1-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:ed957044e368ed295257ae3d212b95456bd9756df490e1ac4538857f67531fcc", size = 4591247, upload-time = "2025-09-17T00:10:02.874Z" }, + { url = "https://files.pythonhosted.org/packages/f6/22/9f3134ae436b63b463cfdf0ff506a0570da6873adb4bf8c19b8a5b4bac64/cryptography-46.0.1-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f7de12fa0eee6234de9a9ce0ffcfa6ce97361db7a50b09b65c63ac58e5f22fc7", size = 4428534, upload-time = "2025-09-17T00:10:04.994Z" }, + { url = "https://files.pythonhosted.org/packages/89/39/e6042bcb2638650b0005c752c38ea830cbfbcbb1830e4d64d530000aa8dc/cryptography-46.0.1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7fab1187b6c6b2f11a326f33b036f7168f5b996aedd0c059f9738915e4e8f53a", size = 4699541, upload-time = "2025-09-17T00:10:06.925Z" }, + { url = "https://files.pythonhosted.org/packages/68/46/753d457492d15458c7b5a653fc9a84a1c9c7a83af6ebdc94c3fc373ca6e8/cryptography-46.0.1-cp38-abi3-win32.whl", hash = "sha256:45f790934ac1018adeba46a0f7289b2b8fe76ba774a88c7f1922213a56c98bc1", size = 3043779, upload-time = "2025-09-17T00:10:08.951Z" }, + { url = "https://files.pythonhosted.org/packages/2f/50/b6f3b540c2f6ee712feeb5fa780bb11fad76634e71334718568e7695cb55/cryptography-46.0.1-cp38-abi3-win_amd64.whl", hash = "sha256:7176a5ab56fac98d706921f6416a05e5aff7df0e4b91516f450f8627cda22af3", size = 3517226, upload-time = "2025-09-17T00:10:10.769Z" }, + { url = "https://files.pythonhosted.org/packages/ff/e8/77d17d00981cdd27cc493e81e1749a0b8bbfb843780dbd841e30d7f50743/cryptography-46.0.1-cp38-abi3-win_arm64.whl", hash = "sha256:efc9e51c3e595267ff84adf56e9b357db89ab2279d7e375ffcaf8f678606f3d9", size = 2923149, upload-time = "2025-09-17T00:10:13.236Z" }, + { url = "https://files.pythonhosted.org/packages/14/b9/b260180b31a66859648cfed5c980544ee22b15f8bd20ef82a23f58c0b83e/cryptography-46.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:fd4b5e2ee4e60425711ec65c33add4e7a626adef79d66f62ba0acfd493af282d", size = 3714683, upload-time = "2025-09-17T00:10:15.601Z" }, + { url = "https://files.pythonhosted.org/packages/c5/5a/1cd3ef86e5884edcbf8b27c3aa8f9544e9b9fcce5d3ed8b86959741f4f8e/cryptography-46.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:48948940d0ae00483e85e9154bb42997d0b77c21e43a77b7773c8c80de532ac5", size = 3443784, upload-time = "2025-09-17T00:10:18.014Z" }, + { url = "https://files.pythonhosted.org/packages/27/27/077e09fd92075dd1338ea0ffaf5cfee641535545925768350ad90d8c36ca/cryptography-46.0.1-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b9c79af2c3058430d911ff1a5b2b96bbfe8da47d5ed961639ce4681886614e70", size = 3722319, upload-time = "2025-09-17T00:10:20.273Z" }, + { url = "https://files.pythonhosted.org/packages/db/32/6fc7250280920418651640d76cee34d91c1e0601d73acd44364570cf041f/cryptography-46.0.1-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:0ca4be2af48c24df689a150d9cd37404f689e2968e247b6b8ff09bff5bcd786f", size = 4249030, upload-time = "2025-09-17T00:10:22.396Z" }, + { url = "https://files.pythonhosted.org/packages/32/33/8d5398b2da15a15110b2478480ab512609f95b45ead3a105c9a9c76f9980/cryptography-46.0.1-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:13e67c4d3fb8b6bc4ef778a7ccdd8df4cd15b4bcc18f4239c8440891a11245cc", size = 4528009, upload-time = "2025-09-17T00:10:24.418Z" }, + { url = "https://files.pythonhosted.org/packages/fd/1c/4012edad2a8977ab386c36b6e21f5065974d37afa3eade83a9968cba4855/cryptography-46.0.1-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:15b5fd9358803b0d1cc42505a18d8bca81dabb35b5cfbfea1505092e13a9d96d", size = 4248902, upload-time = "2025-09-17T00:10:26.255Z" }, + { url = "https://files.pythonhosted.org/packages/58/a3/257cd5ae677302de8fa066fca9de37128f6729d1e63c04dd6a15555dd450/cryptography-46.0.1-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:e34da95e29daf8a71cb2841fd55df0511539a6cdf33e6f77c1e95e44006b9b46", size = 4527150, upload-time = "2025-09-17T00:10:28.28Z" }, + { url = "https://files.pythonhosted.org/packages/6a/cd/fe6b65e1117ec7631f6be8951d3db076bac3e1b096e3e12710ed071ffc3c/cryptography-46.0.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:34f04b7311174469ab3ac2647469743720f8b6c8b046f238e5cb27905695eb2a", size = 3448210, upload-time = "2025-09-17T00:10:30.145Z" }, ] [[package]] @@ -1450,7 +1495,9 @@ name = "networkx" version = "3.5" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy'", + "python_full_version >= '3.12' and python_full_version < '3.14' and platform_python_implementation != 'PyPy'", + "python_full_version >= '3.12' and platform_python_implementation == 'PyPy'", "python_full_version == '3.11.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" } @@ -1537,7 +1584,9 @@ name = "numpy" version = "2.3.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy'", + "python_full_version >= '3.12' and python_full_version < '3.14' and platform_python_implementation != 'PyPy'", + "python_full_version >= '3.12' and platform_python_implementation == 'PyPy'", "python_full_version == '3.11.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/2e/19/d7c972dfe90a353dbd3efbbe1d14a5951de80c99c9dc1b93cd998d51dc0f/numpy-2.3.1.tar.gz", hash = "sha256:1ec9ae20a4226da374362cca3c62cd753faf2f951440b0e3b98e93c235441d2b", size = 20390372, upload-time = "2025-06-21T12:28:33.469Z" } @@ -1814,7 +1863,7 @@ wheels = [ [[package]] name = "pipelex" -version = "0.9.4" +version = "0.10.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiofiles" }, @@ -1845,9 +1894,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "yattag" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/27/99/1e4b5fbfb516f0805a41b530f78da932241141831b4526ab7c1748ede615/pipelex-0.9.4.tar.gz", hash = "sha256:9f3165a3b7edc4fd0e9eea75fc0a3aec17231226b841da379d8ec55068ee9e5a", size = 217199, upload-time = "2025-09-06T13:40:17.296Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c1/a4/bbbf65a7884f90bb4db3bdb05c4c4e781d634f9ca3281526e9601eebea0d/pipelex-0.10.2.tar.gz", hash = "sha256:35fa0c9934574b77d4e622fc22af36c250ce4ef1b0ddbb3a948f65cf60f79edd", size = 219534, upload-time = "2025-09-18T10:37:37.588Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/bd/fd7ef4a63439528eee427f029f8b4b218ef46ec8c71cc97193165801520a/pipelex-0.9.4-py3-none-any.whl", hash = "sha256:cf287bd759e68dba77218ef3ca5ba71fa04895d04a42207f2b689afcbd5fef89", size = 351963, upload-time = "2025-09-06T13:40:15.149Z" }, + { url = "https://files.pythonhosted.org/packages/ef/b3/7529b5780e75a611ac7ef4c15783ff5e888b1e43661893bc8c73b86a146c/pipelex-0.10.2-py3-none-any.whl", hash = "sha256:732a91ca2978c41b120806406b84297532877842ebd8eb4e38c3ecda32c2f668", size = 356816, upload-time = "2025-09-18T10:37:35.672Z" }, ] [package.optional-dependencies] @@ -2021,11 +2070,11 @@ wheels = [ [[package]] name = "pycparser" -version = "2.22" +version = "2.23" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736, upload-time = "2024-03-30T13:22:22.564Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/cf/d2d3b9f5699fb1e4615c8e32ff220203e43b248e1dfcc6736ad9057731ca/pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2", size = 173734, upload-time = "2025-09-09T13:23:47.91Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload-time = "2024-03-30T13:22:20.476Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934", size = 118140, upload-time = "2025-09-09T13:23:46.651Z" }, ] [[package]] @@ -2172,22 +2221,39 @@ wheels = [ [[package]] name = "pynacl" -version = "1.5.0" +version = "1.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi" }, + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a7/22/27582568be639dfe22ddb3902225f91f2f17ceff88ce80e4db396c8986da/PyNaCl-1.5.0.tar.gz", hash = "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba", size = 3392854, upload-time = "2022-01-07T22:05:41.134Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/75/0b8ede18506041c0bf23ac4d8e2971b4161cd6ce630b177d0a08eb0d8857/PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1", size = 349920, upload-time = "2022-01-07T22:05:49.156Z" }, - { url = "https://files.pythonhosted.org/packages/59/bb/fddf10acd09637327a97ef89d2a9d621328850a72f1fdc8c08bdf72e385f/PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92", size = 601722, upload-time = "2022-01-07T22:05:50.989Z" }, - { url = "https://files.pythonhosted.org/packages/5d/70/87a065c37cca41a75f2ce113a5a2c2aa7533be648b184ade58971b5f7ccc/PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394", size = 680087, upload-time = "2022-01-07T22:05:52.539Z" }, - { url = "https://files.pythonhosted.org/packages/ee/87/f1bb6a595f14a327e8285b9eb54d41fef76c585a0edef0a45f6fc95de125/PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d", size = 856678, upload-time = "2022-01-07T22:05:54.251Z" }, - { url = "https://files.pythonhosted.org/packages/66/28/ca86676b69bf9f90e710571b67450508484388bfce09acf8a46f0b8c785f/PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858", size = 1133660, upload-time = "2022-01-07T22:05:56.056Z" }, - { url = "https://files.pythonhosted.org/packages/3d/85/c262db650e86812585e2bc59e497a8f59948a005325a11bbbc9ecd3fe26b/PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b", size = 663824, upload-time = "2022-01-07T22:05:57.434Z" }, - { url = "https://files.pythonhosted.org/packages/fd/1a/cc308a884bd299b651f1633acb978e8596c71c33ca85e9dc9fa33a5399b9/PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff", size = 1117912, upload-time = "2022-01-07T22:05:58.665Z" }, - { url = "https://files.pythonhosted.org/packages/25/2d/b7df6ddb0c2a33afdb358f8af6ea3b8c4d1196ca45497dd37a56f0c122be/PyNaCl-1.5.0-cp36-abi3-win32.whl", hash = "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543", size = 204624, upload-time = "2022-01-07T22:06:00.085Z" }, - { url = "https://files.pythonhosted.org/packages/5e/22/d3db169895faaf3e2eda892f005f433a62db2decbcfbc2f61e6517adfa87/PyNaCl-1.5.0-cp36-abi3-win_amd64.whl", hash = "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93", size = 212141, upload-time = "2022-01-07T22:06:01.861Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/06/c6/a3124dee667a423f2c637cfd262a54d67d8ccf3e160f3c50f622a85b7723/pynacl-1.6.0.tar.gz", hash = "sha256:cb36deafe6e2bce3b286e5d1f3e1c246e0ccdb8808ddb4550bb2792f2df298f2", size = 3505641, upload-time = "2025-09-10T23:39:22.308Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/24/1b639176401255605ba7c2b93a7b1eb1e379e0710eca62613633eb204201/pynacl-1.6.0-cp314-cp314t-macosx_10_10_universal2.whl", hash = "sha256:f46386c24a65383a9081d68e9c2de909b1834ec74ff3013271f1bca9c2d233eb", size = 384141, upload-time = "2025-09-10T23:38:28.675Z" }, + { url = "https://files.pythonhosted.org/packages/5e/7b/874efdf57d6bf172db0df111b479a553c3d9e8bb4f1f69eb3ffff772d6e8/pynacl-1.6.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:dea103a1afcbc333bc0e992e64233d360d393d1e63d0bc88554f572365664348", size = 808132, upload-time = "2025-09-10T23:38:38.995Z" }, + { url = "https://files.pythonhosted.org/packages/f3/61/9b53f5913f3b75ac3d53170cdb897101b2b98afc76f4d9d3c8de5aa3ac05/pynacl-1.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:04f20784083014e265ad58c1b2dd562c3e35864b5394a14ab54f5d150ee9e53e", size = 1407253, upload-time = "2025-09-10T23:38:40.492Z" }, + { url = "https://files.pythonhosted.org/packages/7c/0a/b138916b22bbf03a1bdbafecec37d714e7489dd7bcaf80cd17852f8b67be/pynacl-1.6.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbcc4452a1eb10cd5217318c822fde4be279c9de8567f78bad24c773c21254f8", size = 843719, upload-time = "2025-09-10T23:38:30.87Z" }, + { url = "https://files.pythonhosted.org/packages/01/3b/17c368197dfb2c817ce033f94605a47d0cc27901542109e640cef263f0af/pynacl-1.6.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51fed9fe1bec9e7ff9af31cd0abba179d0e984a2960c77e8e5292c7e9b7f7b5d", size = 1445441, upload-time = "2025-09-10T23:38:33.078Z" }, + { url = "https://files.pythonhosted.org/packages/35/3c/f79b185365ab9be80cd3cd01dacf30bf5895f9b7b001e683b369e0bb6d3d/pynacl-1.6.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:10d755cf2a455d8c0f8c767a43d68f24d163b8fe93ccfaabfa7bafd26be58d73", size = 825691, upload-time = "2025-09-10T23:38:34.832Z" }, + { url = "https://files.pythonhosted.org/packages/f7/1f/8b37d25e95b8f2a434a19499a601d4d272b9839ab8c32f6b0fc1e40c383f/pynacl-1.6.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:536703b8f90e911294831a7fbcd0c062b837f3ccaa923d92a6254e11178aaf42", size = 1410726, upload-time = "2025-09-10T23:38:36.893Z" }, + { url = "https://files.pythonhosted.org/packages/bd/93/5a4a4cf9913014f83d615ad6a2df9187330f764f606246b3a744c0788c03/pynacl-1.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6b08eab48c9669d515a344fb0ef27e2cbde847721e34bba94a343baa0f33f1f4", size = 801035, upload-time = "2025-09-10T23:38:42.109Z" }, + { url = "https://files.pythonhosted.org/packages/bf/60/40da6b0fe6a4d5fd88f608389eb1df06492ba2edca93fca0b3bebff9b948/pynacl-1.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5789f016e08e5606803161ba24de01b5a345d24590a80323379fc4408832d290", size = 1371854, upload-time = "2025-09-10T23:38:44.16Z" }, + { url = "https://files.pythonhosted.org/packages/44/b2/37ac1d65008f824cba6b5bf68d18b76d97d0f62d7a032367ea69d4a187c8/pynacl-1.6.0-cp314-cp314t-win32.whl", hash = "sha256:4853c154dc16ea12f8f3ee4b7e763331876316cc3a9f06aeedf39bcdca8f9995", size = 230345, upload-time = "2025-09-10T23:38:48.276Z" }, + { url = "https://files.pythonhosted.org/packages/f4/5a/9234b7b45af890d02ebee9aae41859b9b5f15fb4a5a56d88e3b4d1659834/pynacl-1.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:347dcddce0b4d83ed3f32fd00379c83c425abee5a9d2cd0a2c84871334eaff64", size = 243103, upload-time = "2025-09-10T23:38:45.503Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2c/c1a0f19d720ab0af3bc4241af2bdf4d813c3ecdcb96392b5e1ddf2d8f24f/pynacl-1.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2d6cd56ce4998cb66a6c112fda7b1fdce5266c9f05044fa72972613bef376d15", size = 187778, upload-time = "2025-09-10T23:38:46.731Z" }, + { url = "https://files.pythonhosted.org/packages/63/37/87c72df19857c5b3b47ace6f211a26eb862ada495cc96daa372d96048fca/pynacl-1.6.0-cp38-abi3-macosx_10_10_universal2.whl", hash = "sha256:f4b3824920e206b4f52abd7de621ea7a44fd3cb5c8daceb7c3612345dfc54f2e", size = 382610, upload-time = "2025-09-10T23:38:49.459Z" }, + { url = "https://files.pythonhosted.org/packages/0c/64/3ce958a5817fd3cc6df4ec14441c43fd9854405668d73babccf77f9597a3/pynacl-1.6.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:16dd347cdc8ae0b0f6187a2608c0af1c8b7ecbbe6b4a06bff8253c192f696990", size = 798744, upload-time = "2025-09-10T23:38:58.531Z" }, + { url = "https://files.pythonhosted.org/packages/e4/8a/3f0dd297a0a33fa3739c255feebd0206bb1df0b44c52fbe2caf8e8bc4425/pynacl-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:16c60daceee88d04f8d41d0a4004a7ed8d9a5126b997efd2933e08e93a3bd850", size = 1397879, upload-time = "2025-09-10T23:39:00.44Z" }, + { url = "https://files.pythonhosted.org/packages/41/94/028ff0434a69448f61348d50d2c147dda51aabdd4fbc93ec61343332174d/pynacl-1.6.0-cp38-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25720bad35dfac34a2bcdd61d9e08d6bfc6041bebc7751d9c9f2446cf1e77d64", size = 833907, upload-time = "2025-09-10T23:38:50.936Z" }, + { url = "https://files.pythonhosted.org/packages/52/bc/a5cff7f8c30d5f4c26a07dfb0bcda1176ab8b2de86dda3106c00a02ad787/pynacl-1.6.0-cp38-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8bfaa0a28a1ab718bad6239979a5a57a8d1506d0caf2fba17e524dbb409441cf", size = 1436649, upload-time = "2025-09-10T23:38:52.783Z" }, + { url = "https://files.pythonhosted.org/packages/7a/20/c397be374fd5d84295046e398de4ba5f0722dc14450f65db76a43c121471/pynacl-1.6.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ef214b90556bb46a485b7da8258e59204c244b1b5b576fb71848819b468c44a7", size = 817142, upload-time = "2025-09-10T23:38:54.4Z" }, + { url = "https://files.pythonhosted.org/packages/12/30/5efcef3406940cda75296c6d884090b8a9aad2dcc0c304daebb5ae99fb4a/pynacl-1.6.0-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:49c336dd80ea54780bcff6a03ee1a476be1612423010472e60af83452aa0f442", size = 1401794, upload-time = "2025-09-10T23:38:56.614Z" }, + { url = "https://files.pythonhosted.org/packages/be/e1/a8fe1248cc17ccb03b676d80fa90763760a6d1247da434844ea388d0816c/pynacl-1.6.0-cp38-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f3482abf0f9815e7246d461fab597aa179b7524628a4bc36f86a7dc418d2608d", size = 772161, upload-time = "2025-09-10T23:39:01.93Z" }, + { url = "https://files.pythonhosted.org/packages/a3/76/8a62702fb657d6d9104ce13449db221a345665d05e6a3fdefb5a7cafd2ad/pynacl-1.6.0-cp38-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:140373378e34a1f6977e573033d1dd1de88d2a5d90ec6958c9485b2fd9f3eb90", size = 1370720, upload-time = "2025-09-10T23:39:03.531Z" }, + { url = "https://files.pythonhosted.org/packages/6d/38/9e9e9b777a1c4c8204053733e1a0269672c0bd40852908c9ad6b6eaba82c/pynacl-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6b393bc5e5a0eb86bb85b533deb2d2c815666665f840a09e0aa3362bb6088736", size = 791252, upload-time = "2025-09-10T23:39:05.058Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d972ce3d92ae05c9091363cf185e8646933f91c376e97b8be79ea6e96c22/pynacl-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4a25cfede801f01e54179b8ff9514bd7b5944da560b7040939732d1804d25419", size = 1362910, upload-time = "2025-09-10T23:39:06.924Z" }, + { url = "https://files.pythonhosted.org/packages/35/2c/ee0b373a1861f66a7ca8bdb999331525615061320dd628527a50ba8e8a60/pynacl-1.6.0-cp38-abi3-win32.whl", hash = "sha256:dcdeb41c22ff3c66eef5e63049abf7639e0db4edee57ba70531fc1b6b133185d", size = 226461, upload-time = "2025-09-10T23:39:11.894Z" }, + { url = "https://files.pythonhosted.org/packages/75/f7/41b6c0b9dd9970173b6acc026bab7b4c187e4e5beef2756d419ad65482da/pynacl-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:cf831615cc16ba324240de79d925eacae8265b7691412ac6b24221db157f6bd1", size = 238802, upload-time = "2025-09-10T23:39:08.966Z" }, + { url = "https://files.pythonhosted.org/packages/8e/0f/462326910c6172fa2c6ed07922b22ffc8e77432b3affffd9e18f444dbfbb/pynacl-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:84709cea8f888e618c21ed9a0efdb1a59cc63141c403db8bf56c469b71ad56f2", size = 183846, upload-time = "2025-09-10T23:39:10.552Z" }, ] [[package]] From c2749b44c8b05b70fb02a93cb31d8ad5b7ba952f Mon Sep 17 00:00:00 2001 From: Louis Choquel Date: Mon, 22 Sep 2025 10:56:33 +0200 Subject: [PATCH 08/10] Last step to remove redundancies in finalized changelog (#50) --- .../pipelines/swe_diff/swe_diff.plx | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/cocode/pipelex_libraries/pipelines/swe_diff/swe_diff.plx b/cocode/pipelex_libraries/pipelines/swe_diff/swe_diff.plx index a9152d2..231ea6c 100644 --- a/cocode/pipelex_libraries/pipelines/swe_diff/swe_diff.plx +++ b/cocode/pipelex_libraries/pipelines/swe_diff/swe_diff.plx @@ -27,6 +27,7 @@ steps = [ { pipe = "draft_changelog_from_git_diff", result = "draft_changelog" }, { pipe = "polish_changelog", result = "structured_changelog" }, { pipe = "format_changelog_as_markdown", result = "markdown_changelog" }, + { pipe = "finalize_changelog" }, ] [pipe.draft_changelog_from_git_diff] @@ -138,3 +139,23 @@ jinja2 = """ {% endif %} """ +[pipe.finalize_changelog] +type = "PipeLLM" +definition = "Polish and improve the changelog" +inputs = { structured_changelog = "StructuredChangelog" } +output = "MarkdownChangelog" +llm = "llm_for_swe" +system_prompt = """ +You are an expert technical writer. Your task is to polish and improve a changelog to make it more clear, concise, and well-structured. +""" +prompt_template = """ +Review and polish the following changelog: + +@structured_changelog + +Remove redundancy: I don't want to see echos between "Changed" and "Fixed" or "Added". +Remove trivial changes. +Keep the markdown formatting and the standard structure of the changelog. +It's OK to remove some sections if they are empty after removing redundancy and trivial changes. +""" + From d06824c8ae4d052abe2b156b7cf51a4a3aee9a7a Mon Sep 17 00:00:00 2001 From: Thomas Hebrard Date: Mon, 22 Sep 2025 12:54:55 +0200 Subject: [PATCH 09/10] fix/clean cli commands (#51) * fix/clean cli commands * update changelog * remove blackbox --- .pipelex/inference/backends.toml | 2 +- CHANGELOG.md | 6 + README.md | 12 +- cocode/cli/__init__.py | 1 + cocode/cli/__main__.py | 8 + cocode/cli/ai_instructions/__init__.py | 1 + .../ai_instructions/ai_instructions_cli.py | 64 ++++ cocode/cli/changelog/__init__.py | 1 + cocode/cli/changelog/changelog_cli.py | 71 ++++ cocode/cli/doc/__init__.py | 1 + cocode/cli/doc/doc_cli.py | 134 +++++++ cocode/cli/features/__init__.py | 1 + cocode/cli/features/features_cli.py | 56 +++ cocode/{cli.py => cli/main.py} | 17 +- cocode/cli/repo/__init__.py | 1 + cocode/cli/repo/repo_cli.py | 86 +++++ cocode/swe/swe_cli.py | 350 ------------------ pyproject.toml | 4 +- uv.lock | 2 +- 19 files changed, 453 insertions(+), 365 deletions(-) create mode 100644 cocode/cli/__init__.py create mode 100644 cocode/cli/__main__.py create mode 100644 cocode/cli/ai_instructions/__init__.py create mode 100644 cocode/cli/ai_instructions/ai_instructions_cli.py create mode 100644 cocode/cli/changelog/__init__.py create mode 100644 cocode/cli/changelog/changelog_cli.py create mode 100644 cocode/cli/doc/__init__.py create mode 100644 cocode/cli/doc/doc_cli.py create mode 100644 cocode/cli/features/__init__.py create mode 100644 cocode/cli/features/features_cli.py rename cocode/{cli.py => cli/main.py} (68%) create mode 100644 cocode/cli/repo/__init__.py create mode 100644 cocode/cli/repo/repo_cli.py delete mode 100644 cocode/swe/swe_cli.py diff --git a/.pipelex/inference/backends.toml b/.pipelex/inference/backends.toml index c3cbc68..cb431c0 100644 --- a/.pipelex/inference/backends.toml +++ b/.pipelex/inference/backends.toml @@ -3,7 +3,7 @@ endpoint = "https://inference.pipelex.com/v1" api_key = "${PIPELEX_INFERENCE_API_KEY}" [blackboxai] -enabled = true +enabled = false endpoint = "https://api.blackbox.ai/v1" api_key = "${BLACKBOX_API_KEY}" diff --git a/CHANGELOG.md b/CHANGELOG.md index a8979c5..08bb920 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [v0.2.3] - 2025-09-22 + +### Changed + +- Cleaned cli commands. + ## [v0.2.2] - 2025-09-18 ### Highlights diff --git a/README.md b/README.md index 06f07b5..b57db2f 100644 --- a/README.md +++ b/README.md @@ -40,16 +40,16 @@ Some complex pipelines require GCP credentials (See [GCP credentials](https://do ### Automatic Documentation & Release Features ```bash # Update documentation based on code changes -cocode swe doc-update v1.0.0 path/to/your/local/repository +cocode doc update v1.0.0 path/to/your/local/repository # Proofread documentation against codebase -cocode swe doc-proofread --doc-dir docs path/to/your/local/repository # Requires GCP credentials for Gemini +cocode doc proofread --doc-dir docs path/to/your/local/repository # Requires GCP credentials for Gemini # Generate changelog from version diff -cocode swe from-repo-diff write_changelog v1.0.0 path/to/your/local/repository # Requires Anthropic API key for claude +cocode changelog update v1.0.0 path/to/your/local/repository # Requires Anthropic API key for claude # Update AI instructions (AGENTS.md, CLAUDE.md, cursor rules) based on code changes -cocode swe ai-instruction-update v1.0.0 path/to/your/local/repository +cocode ai_instructions update v1.0.0 path/to/your/local/repository # GitHub operations cocode github auth # Check GitHub authentication status @@ -151,10 +151,10 @@ cocode repox convert --python-rule imports --output-style import_list #### AI-Powered Analysis ```bash # Extract project fundamentals -cocode swe from-repo extract_fundamentals . --output-filename overview.json +cocode repo extract_fundamentals . --output-filename overview.json # Generate feature documentation -cocode swe from-file extract_features_recap ./analysis.txt --output-filename features.md +cocode features extract ./analysis.txt --output-filename features.md ``` ## 🔧 Configuration diff --git a/cocode/cli/__init__.py b/cocode/cli/__init__.py new file mode 100644 index 0000000..5530a9c --- /dev/null +++ b/cocode/cli/__init__.py @@ -0,0 +1 @@ +"""CLI modules for cocode.""" diff --git a/cocode/cli/__main__.py b/cocode/cli/__main__.py new file mode 100644 index 0000000..1064013 --- /dev/null +++ b/cocode/cli/__main__.py @@ -0,0 +1,8 @@ +""" +Entry point for running cocode CLI as a module. +""" + +from cocode.cli.main import app + +if __name__ == "__main__": + app() diff --git a/cocode/cli/ai_instructions/__init__.py b/cocode/cli/ai_instructions/__init__.py new file mode 100644 index 0000000..189393e --- /dev/null +++ b/cocode/cli/ai_instructions/__init__.py @@ -0,0 +1 @@ +"""AI instructions management module.""" diff --git a/cocode/cli/ai_instructions/ai_instructions_cli.py b/cocode/cli/ai_instructions/ai_instructions_cli.py new file mode 100644 index 0000000..88e64a4 --- /dev/null +++ b/cocode/cli/ai_instructions/ai_instructions_cli.py @@ -0,0 +1,64 @@ +""" +AI instructions management CLI commands. +""" + +import asyncio +from typing import Annotated, List, Optional + +import typer +from pipelex.hub import get_pipeline_tracker + +from cocode.common import validate_repo_path +from cocode.swe.swe_cmd import swe_ai_instruction_update_from_diff + +ai_instructions_app = typer.Typer( + name="ai_instructions", + help="AI instructions management and update commands", + add_completion=False, + rich_markup_mode="rich", + no_args_is_help=True, +) + + +@ai_instructions_app.command("update") +def ai_instructions_update_cmd( + version: Annotated[ + str, + typer.Argument(help="Git version/tag/commit to compare current version against"), + ], + repo_path: Annotated[ + str, + typer.Argument(help="Repository path (local directory) or GitHub URL/identifier (owner/repo or https://github.com/owner/repo)"), + ] = ".", + output_dir: Annotated[ + str, + typer.Option("--output-dir", "-o", help="Output directory path"), + ] = "results", + output_filename: Annotated[ + str, + typer.Option("--output-filename", "-n", help="Output filename"), + ] = "ai-instruction-update-suggestions.txt", + ignore_patterns: Annotated[ + Optional[List[str]], + typer.Option( + "--ignore-pattern", "-i", help="Patterns to exclude from git diff (e.g., '*.log', 'temp/', 'build/'). Can be specified multiple times." + ), + ] = None, +) -> None: + """ + Generate AI instruction update suggestions for AGENTS.md, CLAUDE.md, and cursor rules based on git diff analysis. + Supports both local repositories and GitHub repositories. + """ + repo_path = validate_repo_path(repo_path) + + asyncio.run( + swe_ai_instruction_update_from_diff( + repo_path=repo_path, + version=version, + output_filename=output_filename, + output_dir=output_dir, + ignore_patterns=ignore_patterns, + ) + ) + + get_pipeline_tracker().output_flowchart() diff --git a/cocode/cli/changelog/__init__.py b/cocode/cli/changelog/__init__.py new file mode 100644 index 0000000..620b010 --- /dev/null +++ b/cocode/cli/changelog/__init__.py @@ -0,0 +1 @@ +"""Changelog management module.""" diff --git a/cocode/cli/changelog/changelog_cli.py b/cocode/cli/changelog/changelog_cli.py new file mode 100644 index 0000000..e22b3dd --- /dev/null +++ b/cocode/cli/changelog/changelog_cli.py @@ -0,0 +1,71 @@ +""" +Changelog management CLI commands. +""" + +import asyncio +from typing import Annotated, List, Optional + +import typer +from pipelex.core.pipes.pipe_run_params import PipeRunMode +from pipelex.hub import get_pipeline_tracker + +from cocode.common import get_output_dir, validate_repo_path +from cocode.swe.swe_cmd import swe_from_repo_diff + +changelog_app = typer.Typer( + name="changelog", + help="Changelog management and generation commands", + add_completion=False, + rich_markup_mode="rich", + no_args_is_help=True, +) + + +@changelog_app.command("update") +def changelog_update_cmd( + version: Annotated[ + str, + typer.Argument(help="Git version/tag/commit to compare current version against"), + ], + repo_path: Annotated[ + str, + typer.Argument(help="Repository path (local directory) or GitHub URL/identifier (owner/repo or https://github.com/owner/repo)"), + ] = ".", + output_dir: Annotated[ + Optional[str], + typer.Option("--output-dir", "-o", help="Output directory path. Use 'stdout' to print to console. Defaults to config value if not provided"), + ] = None, + output_filename: Annotated[ + str, + typer.Option("--output-filename", "-n", help="Output filename"), + ] = "changelog-update.md", + dry_run: Annotated[ + bool, + typer.Option("--dry", help="Run pipeline in dry mode (no actual execution)"), + ] = False, + ignore_patterns: Annotated[ + Optional[List[str]], + typer.Option( + "--ignore-pattern", "-i", help="Patterns to exclude from git diff (e.g., '*.log', 'temp/', 'build/'). Can be specified multiple times." + ), + ] = None, +) -> None: + """Generate changelog from git diff comparing current version to specified version. Supports both local repositories and GitHub repositories.""" + repo_path = validate_repo_path(repo_path) + output_dir = get_output_dir(output_dir) + to_stdout = output_dir == "stdout" + pipe_run_mode = PipeRunMode.DRY if dry_run else PipeRunMode.LIVE + + asyncio.run( + swe_from_repo_diff( + pipe_code="write_changelog", + repo_path=repo_path, + version=version, + output_filename=output_filename, + output_dir=output_dir, + to_stdout=to_stdout, + pipe_run_mode=pipe_run_mode, + ignore_patterns=ignore_patterns, + ) + ) + get_pipeline_tracker().output_flowchart() diff --git a/cocode/cli/doc/__init__.py b/cocode/cli/doc/__init__.py new file mode 100644 index 0000000..dc0e813 --- /dev/null +++ b/cocode/cli/doc/__init__.py @@ -0,0 +1 @@ +"""Documentation management module.""" diff --git a/cocode/cli/doc/doc_cli.py b/cocode/cli/doc/doc_cli.py new file mode 100644 index 0000000..baba0e7 --- /dev/null +++ b/cocode/cli/doc/doc_cli.py @@ -0,0 +1,134 @@ +""" +Documentation management CLI commands. +""" + +import asyncio +from typing import Annotated, List, Optional + +import typer +from pipelex.hub import get_pipeline_tracker + +from cocode.common import validate_repo_path +from cocode.swe.swe_cmd import swe_doc_proofread, swe_doc_update_from_diff + +doc_app = typer.Typer( + name="doc", + help="Documentation management and automation commands", + add_completion=False, + rich_markup_mode="rich", + no_args_is_help=True, +) + + +@doc_app.command("update") +def doc_update_cmd( + version: Annotated[ + str, + typer.Argument(help="Git version/tag/commit to compare current version against"), + ], + repo_path: Annotated[ + str, + typer.Argument(help="Repository path (local directory) or GitHub URL/identifier (owner/repo or https://github.com/owner/repo)"), + ] = ".", + output_dir: Annotated[ + str, + typer.Option("--output-dir", "-o", help="Output directory path"), + ] = "results", + output_filename: Annotated[ + str, + typer.Option("--output-filename", "-n", help="Output filename"), + ] = "doc-update-suggestions.txt", + ignore_patterns: Annotated[ + Optional[List[str]], + typer.Option( + "--ignore-pattern", "-i", help="Patterns to exclude from git diff (e.g., '*.log', 'temp/', 'build/'). Can be specified multiple times." + ), + ] = None, + doc_dir: Annotated[ + Optional[str], + typer.Option("--doc-dir", "-d", help="Directory containing documentation files (e.g., 'docs', 'documentation')"), + ] = None, +) -> None: + """ + Generate documentation update suggestions for docs/ directory based on git diff analysis. + Supports both local repositories and GitHub repositories. + """ + repo_path = validate_repo_path(repo_path) + + asyncio.run( + swe_doc_update_from_diff( + repo_path=repo_path, + version=version, + output_filename=output_filename, + output_dir=output_dir, + ignore_patterns=ignore_patterns, + ) + ) + + get_pipeline_tracker().output_flowchart() + + +@doc_app.command("proofread") +def doc_proofread_cmd( + repo_path: Annotated[ + str, + typer.Argument(help="Repository path (local directory) or GitHub URL/identifier (owner/repo or https://github.com/owner/repo)"), + ] = ".", + output_dir: Annotated[ + str, + typer.Option("--output-dir", "-o", help="Output directory path"), + ] = "results", + output_filename: Annotated[ + str, + typer.Option("--output-filename", "-n", help="Output filename"), + ] = "doc-proofread-report", + doc_dir: Annotated[ + str, + typer.Option("--doc-dir", "-d", help="Directory containing documentation files"), + ] = "docs", + include_patterns: Annotated[ + Optional[List[str]], + typer.Option("--include-pattern", "-r", help="Patterns to include in codebase analysis (glob pattern) - can be repeated"), + ] = None, + ignore_patterns: Annotated[ + Optional[List[str]], + typer.Option("--ignore-pattern", "-i", help="Patterns to ignore in codebase analysis (gitignore format) - can be repeated"), + ] = None, +) -> None: + """ + Systematically proofread documentation against actual codebase to find inconsistencies. + Supports both local repositories and GitHub repositories. + """ + repo_path = validate_repo_path(repo_path) + + # Set default include patterns to focus on documentation and code + if include_patterns is None: + include_patterns = ["*.md", "*.py", "*.toml", "*.yaml", "*.yml", "*.json", "*.sh", "*.js", "*.ts"] + + # Set default ignore patterns to exclude noise + if ignore_patterns is None: + ignore_patterns = [ + "__pycache__/", + "*.pyc", + ".git/", + ".venv/", + "node_modules/", + "*.log", + "build/", + "dist/", + ".pytest_cache/", + "*.egg-info/", + ] + + asyncio.run( + swe_doc_proofread( + repo_path=repo_path, + doc_dir=doc_dir, + output_filename=output_filename, + output_dir=output_dir, + include_patterns=include_patterns, + ignore_patterns=ignore_patterns, + ) + ) + + get_pipeline_tracker().output_flowchart() diff --git a/cocode/cli/features/__init__.py b/cocode/cli/features/__init__.py new file mode 100644 index 0000000..61fa3c9 --- /dev/null +++ b/cocode/cli/features/__init__.py @@ -0,0 +1 @@ +"""Feature analysis module.""" diff --git a/cocode/cli/features/features_cli.py b/cocode/cli/features/features_cli.py new file mode 100644 index 0000000..68a0aec --- /dev/null +++ b/cocode/cli/features/features_cli.py @@ -0,0 +1,56 @@ +""" +Feature analysis CLI commands. +""" + +import asyncio +from typing import Annotated, Optional + +import typer +from pipelex.core.pipes.pipe_run_params import PipeRunMode + +from cocode.common import PipeCode, get_output_dir +from cocode.swe.swe_cmd import swe_from_file + +features_app = typer.Typer( + name="features", + help="Feature analysis and extraction commands", + add_completion=False, + rich_markup_mode="rich", + no_args_is_help=True, +) + + +@features_app.command("extract") +def features_extract_cmd( + input_file_path: Annotated[ + str, + typer.Argument(help="Input text file path", exists=True, file_okay=True, dir_okay=False, resolve_path=True), + ], + output_dir: Annotated[ + Optional[str], + typer.Option("--output-dir", "-o", help="Output directory path. Use 'stdout' to print to console. Defaults to config value if not provided"), + ] = None, + output_filename: Annotated[ + str, + typer.Option("--output-filename", "-n", help="Output filename"), + ] = "features.md", + dry_run: Annotated[ + bool, + typer.Option("--dry", help="Run pipeline in dry mode (no actual execution)"), + ] = False, +) -> None: + """Extract and document features from analysis text file.""" + output_dir = get_output_dir(output_dir) + to_stdout = output_dir == "stdout" + pipe_run_mode = PipeRunMode.DRY if dry_run else PipeRunMode.LIVE + + asyncio.run( + swe_from_file( + pipe_code=PipeCode.EXTRACT_FEATURES_RECAP, + input_file_path=input_file_path, + output_filename=output_filename, + output_dir=output_dir, + to_stdout=to_stdout, + pipe_run_mode=pipe_run_mode, + ) + ) diff --git a/cocode/cli.py b/cocode/cli/main.py similarity index 68% rename from cocode/cli.py rename to cocode/cli/main.py index 001f77d..a436530 100644 --- a/cocode/cli.py +++ b/cocode/cli/main.py @@ -11,9 +11,13 @@ from typer.core import TyperGroup from typing_extensions import override +from cocode.cli.ai_instructions.ai_instructions_cli import ai_instructions_app +from cocode.cli.changelog.changelog_cli import changelog_app +from cocode.cli.doc.doc_cli import doc_app +from cocode.cli.features.features_cli import features_app +from cocode.cli.repo.repo_cli import repo_app from cocode.github.github_cli import github_app from cocode.repox.repox_cli import repox_app -from cocode.swe.swe_cli import swe_app from cocode.validation_cli import validation_app @@ -31,8 +35,7 @@ def get_command(self, ctx: Context, cmd_name: str) -> Optional[Command]: app = typer.Typer( name="cocode", help=""" - 🚀 CoCode - Repository Analysis and SWE Automation Tool - + 🚀 CoCode - Repository Analysis and SWE Automation Tool. Convert repository structure and contents to text files for analysis, and perform Software Engineering (SWE) analysis using AI pipelines. @@ -46,8 +49,12 @@ def get_command(self, ctx: Context, cmd_name: str) -> Optional[Command]: ) # Add command groups +app.add_typer(doc_app, name="doc", help="Documentation management and automation commands") +app.add_typer(changelog_app, name="changelog", help="Changelog generation and management commands") +app.add_typer(ai_instructions_app, name="ai_instructions", help="AI instructions update and management commands") +app.add_typer(repo_app, name="repo", help="Repository analysis and processing commands") +app.add_typer(features_app, name="features", help="Feature analysis and extraction commands") app.add_typer(repox_app, name="repox", help="Repository processing and analysis commands") -app.add_typer(swe_app, name="swe", help="Software Engineering analysis and automation commands") app.add_typer(validation_app, name="validation", help="Pipeline validation and setup commands") app.add_typer(github_app, name="github", help="GitHub-related operations and utilities") @@ -55,7 +62,7 @@ def get_command(self, ctx: Context, cmd_name: str) -> Optional[Command]: @app.callback(invoke_without_command=True) def main(ctx: TyperContext) -> None: """Initialize Pipelex system before any command runs.""" - Pipelex.make(relative_config_folder_path="./pipelex_libraries") + Pipelex.make(relative_config_folder_path="../pipelex_libraries") if ctx.invoked_subcommand is None: print(ctx.get_help()) diff --git a/cocode/cli/repo/__init__.py b/cocode/cli/repo/__init__.py new file mode 100644 index 0000000..2c3b806 --- /dev/null +++ b/cocode/cli/repo/__init__.py @@ -0,0 +1 @@ +"""Repository analysis module.""" diff --git a/cocode/cli/repo/repo_cli.py b/cocode/cli/repo/repo_cli.py new file mode 100644 index 0000000..f9477a8 --- /dev/null +++ b/cocode/cli/repo/repo_cli.py @@ -0,0 +1,86 @@ +""" +Repository analysis CLI commands. +""" + +import asyncio +from typing import Annotated, List, Optional + +import typer +from pipelex.core.pipes.pipe_run_params import PipeRunMode + +from cocode.common import PipeCode, get_output_dir, validate_repo_path +from cocode.repox.models import OutputStyle +from cocode.repox.process_python import PythonProcessingRule +from cocode.swe.swe_cmd import swe_from_repo + +repo_app = typer.Typer( + name="repo", + help="Repository analysis and processing commands", + add_completion=False, + rich_markup_mode="rich", + no_args_is_help=True, +) + + +@repo_app.command("extract_fundamentals") +def repo_extract_fundamentals_cmd( + repo_path: Annotated[ + str, + typer.Argument(help="Repository path (local directory) or GitHub URL/identifier (owner/repo or https://github.com/owner/repo)"), + ] = ".", + output_dir: Annotated[ + Optional[str], + typer.Option("--output-dir", "-o", help="Output directory path. Use 'stdout' to print to console. Defaults to config value if not provided"), + ] = None, + output_filename: Annotated[ + str, + typer.Option("--output-filename", "-n", help="Output filename"), + ] = "fundamentals.json", + ignore_patterns: Annotated[ + Optional[List[str]], + typer.Option("--ignore-pattern", "-i", help="List of patterns to ignore (in gitignore format)"), + ] = None, + python_processing_rule: Annotated[ + PythonProcessingRule, + typer.Option("--python-rule", "-p", help="Python processing rule to apply", case_sensitive=False), + ] = PythonProcessingRule.INTERFACE, + output_style: Annotated[ + OutputStyle, + typer.Option( + "--output-style", "-s", help="One of: repo_map, flat (contents only), or import_list (for --python-rule imports)", case_sensitive=False + ), + ] = OutputStyle.REPO_MAP, + include_patterns: Annotated[ + Optional[List[str]], + typer.Option("--include-pattern", "-r", help="Optional pattern to filter files in the tree structure (glob pattern) - can be repeated"), + ] = None, + path_pattern: Annotated[ + Optional[str], + typer.Option("--path-pattern", "-pp", help="Optional pattern to filter paths in the tree structure (regex pattern)"), + ] = None, + dry_run: Annotated[ + bool, + typer.Option("--dry", help="Run pipeline in dry mode (no actual execution)"), + ] = False, +) -> None: + """Extract project fundamentals and architecture insights from repository. Supports both local repositories and GitHub repositories.""" + repo_path = validate_repo_path(repo_path) + output_dir = get_output_dir(output_dir) + to_stdout = output_dir == "stdout" + pipe_run_mode = PipeRunMode.DRY if dry_run else PipeRunMode.LIVE + + asyncio.run( + swe_from_repo( + pipe_code=PipeCode.EXTRACT_FUNDAMENTALS, + repo_path=repo_path, + ignore_patterns=ignore_patterns, + include_patterns=include_patterns, + path_pattern=path_pattern, + python_processing_rule=python_processing_rule, + output_style=output_style, + output_filename=output_filename, + output_dir=output_dir, + to_stdout=to_stdout, + pipe_run_mode=pipe_run_mode, + ) + ) diff --git a/cocode/swe/swe_cli.py b/cocode/swe/swe_cli.py deleted file mode 100644 index 833db5a..0000000 --- a/cocode/swe/swe_cli.py +++ /dev/null @@ -1,350 +0,0 @@ -""" -Software Engineering analysis CLI commands. -""" - -import asyncio -from typing import Annotated, List, Optional - -import typer -from pipelex.core.pipes.pipe_run_params import PipeRunMode -from pipelex.hub import get_pipeline_tracker - -from cocode.common import PipeCode, get_output_dir, get_pipe_descriptions, validate_repo_path -from cocode.repox.models import OutputStyle -from cocode.repox.process_python import PythonProcessingRule - -from .swe_cmd import ( - swe_ai_instruction_update_from_diff, - swe_doc_proofread, - swe_doc_update_from_diff, - swe_from_file, - swe_from_repo, - swe_from_repo_diff, -) - -swe_app = typer.Typer( - name="swe", - help="Software Engineering analysis and automation commands", - add_completion=False, - rich_markup_mode="rich", -) - - -@swe_app.command("from-repo") -def swe_from_repo_cmd( - pipe_code: Annotated[ - PipeCode, - typer.Argument(help=f"Pipeline code to execute for SWE analysis.\n\n{get_pipe_descriptions()}"), - ] = PipeCode.EXTRACT_ONBOARDING_DOCUMENTATION, - repo_path: Annotated[ - str, - typer.Argument(help="Repository path (local directory) or GitHub URL/identifier (owner/repo or https://github.com/owner/repo)"), - ] = ".", - output_dir: Annotated[ - Optional[str], - typer.Option("--output-dir", "-o", help="Output directory path. Use 'stdout' to print to console. Defaults to config value if not provided"), - ] = None, - output_filename: Annotated[ - str, - typer.Option("--output-filename", "-n", help="Output filename"), - ] = "swe-analysis.txt", - ignore_patterns: Annotated[ - Optional[List[str]], - typer.Option("--ignore-pattern", "-i", help="List of patterns to ignore (in gitignore format)"), - ] = None, - python_processing_rule: Annotated[ - PythonProcessingRule, - typer.Option("--python-rule", "-p", help="Python processing rule to apply", case_sensitive=False), - ] = PythonProcessingRule.INTERFACE, - output_style: Annotated[ - OutputStyle, - typer.Option( - "--output-style", "-s", help="One of: repo_map, flat (contents only), or import_list (for --python-rule imports)", case_sensitive=False - ), - ] = OutputStyle.REPO_MAP, - include_patterns: Annotated[ - Optional[List[str]], - typer.Option("--include-pattern", "-r", help="Optional pattern to filter files in the tree structure (glob pattern) - can be repeated"), - ] = None, - path_pattern: Annotated[ - Optional[str], - typer.Option("--path-pattern", "-pp", help="Optional pattern to filter paths in the tree structure (regex pattern)"), - ] = None, - dry_run: Annotated[ - bool, - typer.Option("--dry", help="Run pipeline in dry mode (no actual execution)"), - ] = False, -) -> None: - """Convert repository structure and contents to a text file with SWE analysis. Supports both local repositories and GitHub repositories.""" - repo_path = validate_repo_path(repo_path) - output_dir = get_output_dir(output_dir) - to_stdout = output_dir == "stdout" - pipe_run_mode = PipeRunMode.DRY if dry_run else PipeRunMode.LIVE - - asyncio.run( - swe_from_repo( - pipe_code=pipe_code, - repo_path=repo_path, - ignore_patterns=ignore_patterns, - include_patterns=include_patterns, - path_pattern=path_pattern, - python_processing_rule=python_processing_rule, - output_style=output_style, - output_filename=output_filename, - output_dir=output_dir, - to_stdout=to_stdout, - pipe_run_mode=pipe_run_mode, - ) - ) - - -@swe_app.command("from-file") -def swe_from_file_cmd( - pipe_code: Annotated[ - PipeCode, - typer.Argument(help=f"Pipeline code to execute for SWE analysis.\n\n{get_pipe_descriptions()}"), - ], - input_file_path: Annotated[ - str, - typer.Argument(help="Input text file path", exists=True, file_okay=True, dir_okay=False, resolve_path=True), - ], - output_dir: Annotated[ - Optional[str], - typer.Option("--output-dir", "-o", help="Output directory path. Use 'stdout' to print to console. Defaults to config value if not provided"), - ] = None, - output_filename: Annotated[ - str, - typer.Option("--output-filename", "-n", help="Output filename"), - ] = "swe-analysis.txt", - dry_run: Annotated[ - bool, - typer.Option("--dry", help="Run pipeline in dry mode (no actual execution)"), - ] = False, -) -> None: - """Process SWE analysis from an existing text file.""" - output_dir = get_output_dir(output_dir) - to_stdout = output_dir == "stdout" - pipe_run_mode = PipeRunMode.DRY if dry_run else PipeRunMode.LIVE - - asyncio.run( - swe_from_file( - pipe_code=pipe_code, - input_file_path=input_file_path, - output_filename=output_filename, - output_dir=output_dir, - to_stdout=to_stdout, - pipe_run_mode=pipe_run_mode, - ) - ) - - -@swe_app.command("from-repo-diff") -def swe_from_repo_diff_cmd( - pipe_code: Annotated[ - str, - typer.Argument(help="Pipeline code to execute for SWE analysis"), - ], - version: Annotated[ - str, - typer.Argument(help="Git version/tag/commit to compare current version against"), - ], - repo_path: Annotated[ - str, - typer.Argument(help="Repository path (local directory) or GitHub URL/identifier (owner/repo or https://github.com/owner/repo)"), - ] = ".", - output_dir: Annotated[ - Optional[str], - typer.Option("--output-dir", "-o", help="Output directory path. Use 'stdout' to print to console. Defaults to config value if not provided"), - ] = None, - output_filename: Annotated[ - str, - typer.Option("--output-filename", "-n", help="Output filename"), - ] = "swe-diff-analysis.md", - dry_run: Annotated[ - bool, - typer.Option("--dry", help="Run pipeline in dry mode (no actual execution)"), - ] = False, - ignore_patterns: Annotated[ - Optional[List[str]], - typer.Option( - "--ignore-pattern", "-i", help="Patterns to exclude from git diff (e.g., '*.log', 'temp/', 'build/'). Can be specified multiple times." - ), - ] = None, -) -> None: - """Process SWE analysis from git diff comparing current version to specified version. Supports both local repositories and GitHub repositories.""" - repo_path = validate_repo_path(repo_path) - output_dir = get_output_dir(output_dir) - to_stdout = output_dir == "stdout" - pipe_run_mode = PipeRunMode.DRY if dry_run else PipeRunMode.LIVE - - asyncio.run( - swe_from_repo_diff( - pipe_code=pipe_code, - repo_path=repo_path, - version=version, - output_filename=output_filename, - output_dir=output_dir, - to_stdout=to_stdout, - pipe_run_mode=pipe_run_mode, - ignore_patterns=ignore_patterns, - ) - ) - get_pipeline_tracker().output_flowchart() - - -@swe_app.command("doc-update") -def swe_doc_update_cmd( - version: Annotated[ - str, - typer.Argument(help="Git version/tag/commit to compare current version against"), - ], - repo_path: Annotated[ - str, - typer.Argument(help="Repository path (local directory) or GitHub URL/identifier (owner/repo or https://github.com/owner/repo)"), - ] = ".", - output_dir: Annotated[ - str, - typer.Option("--output-dir", "-o", help="Output directory path"), - ] = "results", - output_filename: Annotated[ - str, - typer.Option("--output-filename", "-n", help="Output filename"), - ] = "doc-update-suggestions.txt", - ignore_patterns: Annotated[ - Optional[List[str]], - typer.Option( - "--ignore-pattern", "-i", help="Patterns to exclude from git diff (e.g., '*.log', 'temp/', 'build/'). Can be specified multiple times." - ), - ] = None, - doc_dir: Annotated[ - Optional[str], - typer.Option("--doc-dir", "-d", help="Directory containing documentation files (e.g., 'docs', 'documentation')"), - ] = None, -) -> None: - """ - Generate documentation update suggestions for docs/ directory based on git diff analysis. - Supports both local repositories and GitHub repositories. - """ - repo_path = validate_repo_path(repo_path) - - asyncio.run( - swe_doc_update_from_diff( - repo_path=repo_path, - version=version, - output_filename=output_filename, - output_dir=output_dir, - ignore_patterns=ignore_patterns, - ) - ) - - get_pipeline_tracker().output_flowchart() - - -@swe_app.command("ai-instruction-update") -def swe_ai_instruction_update_cmd( - version: Annotated[ - str, - typer.Argument(help="Git version/tag/commit to compare current version against"), - ], - repo_path: Annotated[ - str, - typer.Argument(help="Repository path (local directory) or GitHub URL/identifier (owner/repo or https://github.com/owner/repo)"), - ] = ".", - output_dir: Annotated[ - str, - typer.Option("--output-dir", "-o", help="Output directory path"), - ] = "results", - output_filename: Annotated[ - str, - typer.Option("--output-filename", "-n", help="Output filename"), - ] = "ai-instruction-update-suggestions.txt", - ignore_patterns: Annotated[ - Optional[List[str]], - typer.Option( - "--ignore-pattern", "-i", help="Patterns to exclude from git diff (e.g., '*.log', 'temp/', 'build/'). Can be specified multiple times." - ), - ] = None, -) -> None: - """ - Generate AI instruction update suggestions for AGENTS.md, CLAUDE.md, and cursor rules based on git diff analysis. - Supports both local repositories and GitHub repositories. - """ - repo_path = validate_repo_path(repo_path) - - asyncio.run( - swe_ai_instruction_update_from_diff( - repo_path=repo_path, - version=version, - output_filename=output_filename, - output_dir=output_dir, - ignore_patterns=ignore_patterns, - ) - ) - - get_pipeline_tracker().output_flowchart() - - -@swe_app.command("doc-proofread") -def swe_doc_proofread_cmd( - repo_path: Annotated[ - str, - typer.Argument(help="Repository path (local directory) or GitHub URL/identifier (owner/repo or https://github.com/owner/repo)"), - ] = ".", - output_dir: Annotated[ - str, - typer.Option("--output-dir", "-o", help="Output directory path"), - ] = "results", - output_filename: Annotated[ - str, - typer.Option("--output-filename", "-n", help="Output filename"), - ] = "doc-proofread-report", - doc_dir: Annotated[ - str, - typer.Option("--doc-dir", "-d", help="Directory containing documentation files"), - ] = "docs", - include_patterns: Annotated[ - Optional[List[str]], - typer.Option("--include-pattern", "-r", help="Patterns to include in codebase analysis (glob pattern) - can be repeated"), - ] = None, - ignore_patterns: Annotated[ - Optional[List[str]], - typer.Option("--ignore-pattern", "-i", help="Patterns to ignore in codebase analysis (gitignore format) - can be repeated"), - ] = None, -) -> None: - """ - Systematically proofread documentation against actual codebase to find inconsistencies. - Supports both local repositories and GitHub repositories. - """ - repo_path = validate_repo_path(repo_path) - - # Set default include patterns to focus on documentation and code - if include_patterns is None: - include_patterns = ["*.md", "*.py", "*.toml", "*.yaml", "*.yml", "*.json", "*.sh", "*.js", "*.ts"] - - # Set default ignore patterns to exclude noise - if ignore_patterns is None: - ignore_patterns = [ - "__pycache__/", - "*.pyc", - ".git/", - ".venv/", - "node_modules/", - "*.log", - "build/", - "dist/", - ".pytest_cache/", - "*.egg-info/", - ] - - asyncio.run( - swe_doc_proofread( - repo_path=repo_path, - doc_dir=doc_dir, - output_filename=output_filename, - output_dir=output_dir, - include_patterns=include_patterns, - ignore_patterns=ignore_patterns, - ) - ) - - get_pipeline_tracker().output_flowchart() diff --git a/pyproject.toml b/pyproject.toml index 8bceee6..0174231 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "cocode" -version = "0.2.2" +version = "0.2.3" description = "Cocode is the friend of your code" authors = [{ name = "Evotis S.A.S.", email = "evotis@pipelex.com" }] maintainers = [{ name = "Pipelex staff", email = "oss@pipelex.com" }] @@ -47,7 +47,7 @@ dev = [ ] [project.scripts] -cocode = "cocode.cli:app" +cocode = "cocode.cli.main:app" [build-system] requires = ["hatchling"] diff --git a/uv.lock b/uv.lock index dfd877e..5446679 100644 --- a/uv.lock +++ b/uv.lock @@ -510,7 +510,7 @@ wheels = [ [[package]] name = "cocode" -version = "0.2.1" +version = "0.2.3" source = { editable = "." } dependencies = [ { name = "pipelex", extra = ["anthropic", "bedrock", "google"] }, From f67449f00d105c953d643ac1146a6b11561f718f Mon Sep 17 00:00:00 2001 From: Thomas Hebrard Date: Mon, 29 Sep 2025 14:46:46 +0200 Subject: [PATCH 10/10] Fix/clean cli commands (#53) --- .pipelex/inference/deck/cocode_deck.toml | 5 ----- .pipelex/inference/deck/overrides.toml | 5 +++-- .../pipelines/doc_proofread/doc_proofread.plx | 2 ++ cocode/validation_cli.py | 7 ++++--- pyproject.toml | 2 +- tests/integration/test_basic.py | 6 +++--- uv.lock | 2 +- 7 files changed, 14 insertions(+), 15 deletions(-) delete mode 100644 .pipelex/inference/deck/cocode_deck.toml diff --git a/.pipelex/inference/deck/cocode_deck.toml b/.pipelex/inference/deck/cocode_deck.toml deleted file mode 100644 index 1a6ab62..0000000 --- a/.pipelex/inference/deck/cocode_deck.toml +++ /dev/null @@ -1,5 +0,0 @@ -[llm_presets] -llm_for_large_text = { llm_handle = "gemini-2.5-pro", temperature = 0.1 } -# llm_for_swe = { llm_handle = "gpt-4o", temperature = 0.1 } -llm_for_swe = { llm_handle = "claude-4-sonnet", temperature = 0.1 } - diff --git a/.pipelex/inference/deck/overrides.toml b/.pipelex/inference/deck/overrides.toml index 999cc4d..496d0e8 100644 --- a/.pipelex/inference/deck/overrides.toml +++ b/.pipelex/inference/deck/overrides.toml @@ -1,5 +1,3 @@ - - #################################################################################################### # LLM Deck overrides #################################################################################################### @@ -8,3 +6,6 @@ for_text = "disabled" for_object = "disabled" +[llm_presets] +llm_for_large_text = { llm_handle = "gemini-2.5-pro", temperature = 0.1 } +llm_for_swe = { llm_handle = "claude-4-sonnet", temperature = 0.1 } \ No newline at end of file diff --git a/cocode/pipelex_libraries/pipelines/doc_proofread/doc_proofread.plx b/cocode/pipelex_libraries/pipelines/doc_proofread/doc_proofread.plx index 78e0dc1..d022ebd 100644 --- a/cocode/pipelex_libraries/pipelines/doc_proofread/doc_proofread.plx +++ b/cocode/pipelex_libraries/pipelines/doc_proofread/doc_proofread.plx @@ -18,6 +18,8 @@ inputs = { doc_file = "DocumentationFile", repo_map = "RepositoryMap" } output = "FilePath" multiple_output = true llm = "llm_for_large_text" +structuring_method = "preliminary_text" +llm_to_structure = "cheap_llm_for_object" system_prompt = """ Extract code elements mentioned in docs (classes, functions, commands) and find their actual implementations or usages in the codebase. """ diff --git a/cocode/validation_cli.py b/cocode/validation_cli.py index 9ecb2e9..cc6cd66 100644 --- a/cocode/validation_cli.py +++ b/cocode/validation_cli.py @@ -6,7 +6,8 @@ import typer from pipelex import log -from pipelex.pipe_works.pipe_dry import dry_run_all_pipes +from pipelex.hub import get_pipe_provider +from pipelex.pipe_works.pipe_dry import dry_run_pipes from pipelex.pipelex import Pipelex validation_app = typer.Typer( @@ -21,14 +22,14 @@ def validate() -> None: """Run the setup sequence and validate all pipelines.""" Pipelex.get_instance().validate_libraries() - asyncio.run(dry_run_all_pipes()) + asyncio.run(dry_run_pipes(get_pipe_provider().get_pipes())) log.info("Setup sequence passed OK, config and pipelines are validated.") @validation_app.command("dry-run") def dry_run() -> None: """Run dry validation of all pipelines without full setup.""" - asyncio.run(dry_run_all_pipes()) + asyncio.run(dry_run_pipes(get_pipe_provider().get_pipes())) log.info("Dry run completed successfully.") diff --git a/pyproject.toml b/pyproject.toml index 0174231..d5af317 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ "Operating System :: OS Independent", ] -dependencies = ["pipelex[anthropic,google,bedrock]>=0.10.2", "PyGithub==2.4.0"] +dependencies = ["pipelex[anthropic,google,google-genai,bedrock]==0.10.2", "PyGithub==2.4.0"] [project.optional-dependencies] docs = [ diff --git a/tests/integration/test_basic.py b/tests/integration/test_basic.py index 7db5d29..05e3068 100644 --- a/tests/integration/test_basic.py +++ b/tests/integration/test_basic.py @@ -1,7 +1,7 @@ import asyncio -from pipelex.hub import get_required_concept -from pipelex.pipe_works.pipe_dry import dry_run_all_pipes +from pipelex.hub import get_pipe_provider, get_required_concept +from pipelex.pipe_works.pipe_dry import dry_run_pipes def test_boot(): @@ -15,4 +15,4 @@ def test_concept_exists(): def test_dry_run_all_pipes(): """Test that dry_run_all_pipes() runs successfully without errors.""" # This should not raise any exceptions - asyncio.run(dry_run_all_pipes()) + asyncio.run(dry_run_pipes(get_pipe_provider().get_pipes())) diff --git a/uv.lock b/uv.lock index 5446679..1efa55d 100644 --- a/uv.lock +++ b/uv.lock @@ -553,7 +553,7 @@ requires-dist = [ { name = "mkdocs-material", marker = "extra == 'docs'", specifier = "==9.6.14" }, { name = "mkdocs-meta-manager", marker = "extra == 'docs'", specifier = "==1.1.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.11.2" }, - { name = "pipelex", extras = ["anthropic", "google", "bedrock"], specifier = ">=0.10.2" }, + { name = "pipelex", extras = ["anthropic", "google", "google-genai", "bedrock"], specifier = "==0.10.2" }, { name = "pygithub", specifier = "==2.4.0" }, { name = "pyright", marker = "extra == 'dev'", specifier = "==1.1.398" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.3" },