From 6cbb9640f0457449afe62bb6bd1dcc31a978f849 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Fri, 26 Jun 2026 00:45:01 +0100 Subject: [PATCH] Add apache-airflow-providers-anthropic provider Add a dedicated provider for the Anthropic Claude API so Dag authors can use Claude through the official Anthropic Python SDK instead of hand-writing SDK calls in tasks. It offers first-class connection management (first-party API, Amazon Bedrock, Google Vertex AI, Claude Platform on AWS, Microsoft Foundry, and keyless Workload Identity Federation) with labelled connection form fields, plus deferrable Message Batches and Managed Agents session operators, a batch sensor, and matching triggers. Targets Airflow 3+ and ships as not-ready / incubation while the Managed Agents beta surface stabilises. --- .../ISSUE_TEMPLATE/1-airflow_bug_report.yml | 1 + .github/boring-cyborg.yml | 3 + airflow-core/docs/extra-packages-ref.rst | 2 + .../src/airflow/provider.yaml.schema.json | 2 + dev/breeze/doc/images/output-commands.svg | 2 +- dev/breeze/doc/images/output_build-docs.svg | 2 +- dev/breeze/doc/images/output_build-docs.txt | 2 +- ...release-management_add-back-references.svg | 2 +- ...release-management_add-back-references.txt | 2 +- ...e-management_classify-provider-changes.svg | 22 +- ...e-management_classify-provider-changes.txt | 2 +- ...ement_generate-issue-content-providers.svg | 22 +- ...ement_generate-issue-content-providers.txt | 2 +- ...management_generate-providers-metadata.svg | 50 +- ...management_generate-providers-metadata.txt | 2 +- ...agement_prepare-provider-distributions.svg | 22 +- ...agement_prepare-provider-distributions.txt | 2 +- ...agement_prepare-provider-documentation.svg | 22 +- ...agement_prepare-provider-documentation.txt | 2 +- ...output_release-management_publish-docs.svg | 2 +- ...output_release-management_publish-docs.txt | 2 +- ...t_sbom_generate-providers-requirements.svg | 4 +- ...t_sbom_generate-providers-requirements.txt | 2 +- .../output_workflow-run_publish-docs.svg | 2 +- .../output_workflow-run_publish-docs.txt | 2 +- .../src/airflow_breeze/global_constants.py | 2 +- docs/spelling_wordlist.txt | 3 + providers/anthropic/.gitignore | 1 + providers/anthropic/LICENSE | 201 +++++++ providers/anthropic/NOTICE | 5 + providers/anthropic/README.rst | 83 +++ .../docs/.latest-doc-only-change.txt | 1 + providers/anthropic/docs/changelog.rst | 26 + providers/anthropic/docs/commits.rst | 34 ++ providers/anthropic/docs/conf.py | 27 + providers/anthropic/docs/connections.rst | 106 ++++ providers/anthropic/docs/index.rst | 130 ++++ .../installing-providers-from-sources.rst | 18 + .../docs/integration-logos/Anthropic.png | Bin 0 -> 11562 bytes .../anthropic/docs/operators/anthropic.rst | 163 +++++ providers/anthropic/docs/security.rst | 38 ++ providers/anthropic/provider.yaml | 139 +++++ providers/anthropic/pyproject.toml | 134 +++++ providers/anthropic/src/airflow/__init__.py | 17 + .../src/airflow/providers/__init__.py | 17 + .../airflow/providers/anthropic/__init__.py | 39 ++ .../airflow/providers/anthropic/exceptions.py | 37 ++ .../providers/anthropic/get_provider_info.py | 118 ++++ .../providers/anthropic/hooks/__init__.py | 16 + .../providers/anthropic/hooks/anthropic.py | 569 ++++++++++++++++++ .../providers/anthropic/operators/__init__.py | 16 + .../providers/anthropic/operators/agent.py | 219 +++++++ .../anthropic/operators/anthropic.py | 200 ++++++ .../providers/anthropic/sensors/__init__.py | 16 + .../providers/anthropic/sensors/anthropic.py | 124 ++++ .../providers/anthropic/triggers/__init__.py | 16 + .../providers/anthropic/triggers/agent.py | 128 ++++ .../providers/anthropic/triggers/anthropic.py | 113 ++++ providers/anthropic/tests/conftest.py | 19 + providers/anthropic/tests/system/__init__.py | 17 + .../tests/system/anthropic/__init__.py | 16 + .../anthropic/example_anthropic_agent.py | 65 ++ .../anthropic/example_anthropic_batch.py | 81 +++ providers/anthropic/tests/unit/__init__.py | 17 + .../tests/unit/anthropic/__init__.py | 16 + .../tests/unit/anthropic/hooks/__init__.py | 16 + .../unit/anthropic/hooks/test_anthropic.py | 464 ++++++++++++++ .../unit/anthropic/operators/__init__.py | 16 + .../unit/anthropic/operators/test_agent.py | 179 ++++++ .../anthropic/operators/test_anthropic.py | 190 ++++++ .../tests/unit/anthropic/sensors/__init__.py | 16 + .../unit/anthropic/sensors/test_anthropic.py | 104 ++++ .../tests/unit/anthropic/test_exceptions.py | 44 ++ .../tests/unit/anthropic/triggers/__init__.py | 16 + .../unit/anthropic/triggers/test_agent.py | 120 ++++ .../unit/anthropic/triggers/test_anthropic.py | 125 ++++ pyproject.toml | 10 + scripts/ci/docker-compose/remove-sources.yml | 1 + scripts/ci/docker-compose/tests-sources.yml | 1 + uv.lock | 75 ++- 80 files changed, 4458 insertions(+), 86 deletions(-) create mode 100644 providers/anthropic/.gitignore create mode 100644 providers/anthropic/LICENSE create mode 100644 providers/anthropic/NOTICE create mode 100644 providers/anthropic/README.rst create mode 100644 providers/anthropic/docs/.latest-doc-only-change.txt create mode 100644 providers/anthropic/docs/changelog.rst create mode 100644 providers/anthropic/docs/commits.rst create mode 100644 providers/anthropic/docs/conf.py create mode 100644 providers/anthropic/docs/connections.rst create mode 100644 providers/anthropic/docs/index.rst create mode 100644 providers/anthropic/docs/installing-providers-from-sources.rst create mode 100644 providers/anthropic/docs/integration-logos/Anthropic.png create mode 100644 providers/anthropic/docs/operators/anthropic.rst create mode 100644 providers/anthropic/docs/security.rst create mode 100644 providers/anthropic/provider.yaml create mode 100644 providers/anthropic/pyproject.toml create mode 100644 providers/anthropic/src/airflow/__init__.py create mode 100644 providers/anthropic/src/airflow/providers/__init__.py create mode 100644 providers/anthropic/src/airflow/providers/anthropic/__init__.py create mode 100644 providers/anthropic/src/airflow/providers/anthropic/exceptions.py create mode 100644 providers/anthropic/src/airflow/providers/anthropic/get_provider_info.py create mode 100644 providers/anthropic/src/airflow/providers/anthropic/hooks/__init__.py create mode 100644 providers/anthropic/src/airflow/providers/anthropic/hooks/anthropic.py create mode 100644 providers/anthropic/src/airflow/providers/anthropic/operators/__init__.py create mode 100644 providers/anthropic/src/airflow/providers/anthropic/operators/agent.py create mode 100644 providers/anthropic/src/airflow/providers/anthropic/operators/anthropic.py create mode 100644 providers/anthropic/src/airflow/providers/anthropic/sensors/__init__.py create mode 100644 providers/anthropic/src/airflow/providers/anthropic/sensors/anthropic.py create mode 100644 providers/anthropic/src/airflow/providers/anthropic/triggers/__init__.py create mode 100644 providers/anthropic/src/airflow/providers/anthropic/triggers/agent.py create mode 100644 providers/anthropic/src/airflow/providers/anthropic/triggers/anthropic.py create mode 100644 providers/anthropic/tests/conftest.py create mode 100644 providers/anthropic/tests/system/__init__.py create mode 100644 providers/anthropic/tests/system/anthropic/__init__.py create mode 100644 providers/anthropic/tests/system/anthropic/example_anthropic_agent.py create mode 100644 providers/anthropic/tests/system/anthropic/example_anthropic_batch.py create mode 100644 providers/anthropic/tests/unit/__init__.py create mode 100644 providers/anthropic/tests/unit/anthropic/__init__.py create mode 100644 providers/anthropic/tests/unit/anthropic/hooks/__init__.py create mode 100644 providers/anthropic/tests/unit/anthropic/hooks/test_anthropic.py create mode 100644 providers/anthropic/tests/unit/anthropic/operators/__init__.py create mode 100644 providers/anthropic/tests/unit/anthropic/operators/test_agent.py create mode 100644 providers/anthropic/tests/unit/anthropic/operators/test_anthropic.py create mode 100644 providers/anthropic/tests/unit/anthropic/sensors/__init__.py create mode 100644 providers/anthropic/tests/unit/anthropic/sensors/test_anthropic.py create mode 100644 providers/anthropic/tests/unit/anthropic/test_exceptions.py create mode 100644 providers/anthropic/tests/unit/anthropic/triggers/__init__.py create mode 100644 providers/anthropic/tests/unit/anthropic/triggers/test_agent.py create mode 100644 providers/anthropic/tests/unit/anthropic/triggers/test_anthropic.py diff --git a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml index 60c117f437d32..724f5a0b87ce3 100644 --- a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml @@ -115,6 +115,7 @@ body: - akeyless - alibaba - amazon + - anthropic - apache-beam - apache-cassandra - apache-drill diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index f83c78cb39ee2..98e38346e98a3 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -30,6 +30,9 @@ labelPRBasedOnFilePath: provider:amazon: - providers/amazon/** + provider:anthropic: + - providers/anthropic/** + provider:apache-beam: - providers/apache/beam/** diff --git a/airflow-core/docs/extra-packages-ref.rst b/airflow-core/docs/extra-packages-ref.rst index eb0acd31df1cb..c2bb02ba55dbd 100644 --- a/airflow-core/docs/extra-packages-ref.rst +++ b/airflow-core/docs/extra-packages-ref.rst @@ -243,6 +243,8 @@ These are extras that add dependencies needed for integration with external serv +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | amazon | ``pip install 'apache-airflow[amazon]'`` | Amazon Web Services | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ +| anthropic | ``pip install 'apache-airflow[anthropic]'`` | Anthropic hooks and operators | ++---------------------+-----------------------------------------------------+-----------------------------------------------------+ | asana | ``pip install 'apache-airflow[asana]'`` | Asana hooks and operators | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | atlassian-jira | ``pip install 'apache-airflow[atlassian-jira]'`` | Jira hooks and operators | diff --git a/airflow-core/src/airflow/provider.yaml.schema.json b/airflow-core/src/airflow/provider.yaml.schema.json index 50dacbc79634c..b9d12bc29754d 100644 --- a/airflow-core/src/airflow/provider.yaml.schema.json +++ b/airflow-core/src/airflow/provider.yaml.schema.json @@ -99,9 +99,11 @@ "ai", "ai-inference", "alibaba", + "anthropic", "apache", "aws", "azure", + "claude", "dbt", "embeddings", "gcp", diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index 6dfe7e09c52c4..4891c30ed5124 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -367,7 +367,7 @@ -Usage:breeze[OPTIONS] [COMMAND] [ARGS]... +Usage:breeze[OPTIONS] COMMAND [ARGS]... ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--answer -aForce answer to questions. (y | n | q | yes | no | quit)│ diff --git a/dev/breeze/doc/images/output_build-docs.svg b/dev/breeze/doc/images/output_build-docs.svg index 172f0f6576eb2..21272cc62296c 100644 --- a/dev/breeze/doc/images/output_build-docs.svg +++ b/dev/breeze/doc/images/output_build-docs.svg @@ -240,7 +240,7 @@ Usage:                                                                                                                 breeze build-docs                                                                                                      -[OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | apache-airflow | apache-airflow-ctl |               +[OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | anthropic | apache-airflow | apache-airflow-ctl |   apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes diff --git a/dev/breeze/doc/images/output_build-docs.txt b/dev/breeze/doc/images/output_build-docs.txt index a734b81b0c766..48b11286965d5 100644 --- a/dev/breeze/doc/images/output_build-docs.txt +++ b/dev/breeze/doc/images/output_build-docs.txt @@ -1 +1 @@ -de29604dc4d7031b9a26a1e0350d7806 +28c9a42f6270c2fa37c15cca78a8c38b diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.svg b/dev/breeze/doc/images/output_release-management_add-back-references.svg index 88ba1696907ae..814f0d28ea328 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.svg +++ b/dev/breeze/doc/images/output_release-management_add-back-references.svg @@ -149,7 +149,7 @@ Usage:                                                                                                                 breeze release-management add-back-references                                                                          -[OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | apache-airflow | apache-airflow-ctl |               +[OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | anthropic | apache-airflow | apache-airflow-ctl |   apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.txt b/dev/breeze/doc/images/output_release-management_add-back-references.txt index 9d5b0b1d2fef4..8562a83f6d21c 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.txt +++ b/dev/breeze/doc/images/output_release-management_add-back-references.txt @@ -1 +1 @@ -4b44b7c52911adf1c1bdfb87e10e873b +a100217703deb9fddf97c53ef7e959a5 diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg index e7b78f7153ee1..cef6e4d3b1d14 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg @@ -162,17 +162,17 @@ Usage:                                                                                                                 breeze release-management classify-provider-changes                                                                    -[OPTIONS] [airbyte | akeyless | alibaba | amazon | apache.cassandra | apache.drill | apache.druid | apache.flink |     -apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  -apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | -cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql |          -databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp -| git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins | keycloak |        -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres |     -presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |    -sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb |          -zendesk]...                                                                                                            +[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        +apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy +| apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        +common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   +facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    +keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       +openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       +postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    +ydb | zendesk]...                                                                                                      Classify each provider's unreleased changes with hard-coded, high-confidence rules, flagging ambiguous commits as  'needs_llm' for an agent/skill to assess. Outputs JSON - a deterministic alternative to the random '--non-interactive' diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt index 226f5e3c0feab..6e02c700afdf5 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt @@ -1 +1 @@ -57466e5cb9470b0d4398724fda2a1825 +4cac13b21eee8b732a46c5a15aec7a4b diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg index 1ce32a4d94152..2f9692be54fa0 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg @@ -151,17 +151,17 @@ Usage:                                                                                                                 breeze release-management generate-issue-content-providers                                                             -[OPTIONS] [airbyte | akeyless | alibaba | amazon | apache.cassandra | apache.drill | apache.druid | apache.flink |     -apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  -apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | -cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql |          -databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp -| git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins | keycloak |        -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres |     -presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |    -sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb |          -zendesk]...                                                                                                            +[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        +apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy +| apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        +common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   +facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    +keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       +openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       +postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    +ydb | zendesk]...                                                                                                      Generates content for issue to test the release. diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt index 5231c239ad495..6afb289b5854f 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -e2d5118e3339b53ce6e87dc6be6ae053 +bd80bf2dd63a27111b8c5745df5dfb86 diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg index 40358ab62d9ca..64efb0af6b65b 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg @@ -1,4 +1,4 @@ - + │--provider-id     Provider_id to generate metadata for. If not specified, metadata for all packages will be       │ │generated. This is debug-only option. When this option is specified, the metadata produced for  │ │the distribution will not be written to the file, but printed via console. (airbyte | akeyless |│ -│alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink |│ -│apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | │ -│apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb |│ -│asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai│ -│| common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |│ -│dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | │ -│github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins | │ -│keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql │ -│| neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | │ -│papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | │ -│segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | │ -│tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb | zendesk)│ -│--provider-versionProvider version to generate metadata for. Only used when --provider-id is specified. Limits    │ -│running metadata generation to only this version of the provider. (TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help   -hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│alibaba | amazon | anthropic | apache.beam | apache.cassandra | apache.drill | apache.druid | │ +│apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | │ +│apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | │ +│apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes│ +│| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | │ +│datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | │ +│facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica│ +│| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | │ +│microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | │ +│opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | │ +│redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake │ +│| sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |│ +│yandex | ydb | zendesk)│ +│--provider-versionProvider version to generate metadata for. Only used when --provider-id is specified. Limits    │ +│running metadata generation to only this version of the provider. (TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help   -hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt index 088e7af08dc1f..7c14e04e63151 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt @@ -1 +1 @@ -93f9b27ce19f188a7d7b70cfcabf8c2b +5865810f5a4b12c9199267b7dd6459b3 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg index 3baea7f764abe..27a1d92de9a4b 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg @@ -189,17 +189,17 @@ Usage:                                                                                                                 breeze release-management prepare-provider-distributions                                                               -[OPTIONS] [airbyte | akeyless | alibaba | amazon | apache.cassandra | apache.drill | apache.druid | apache.flink |     -apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  -apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | -cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql |          -databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp -| git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins | keycloak |        -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres |     -presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |    -sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb |          -zendesk]...                                                                                                            +[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        +apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy +| apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        +common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   +facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    +keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       +openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       +postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    +ydb | zendesk]...                                                                                                      Prepare sdist/whl distributions of Airflow Providers. Each provider directory is wiped with `git clean -fdx`  (preserving .venv, .idea, .vscode) before build to keep in-tree generated files out of the artifact. See dev/breeze  diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt index a4104f3c3a4d2..f054c03806e05 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt @@ -1 +1 @@ -3581abc65c55d4b8a096005c312df783 +a27c1726f5902e5fdb501ecdee226476 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg index 571cc1a3c80e1..b9c71f9c81aec 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg @@ -210,17 +210,17 @@ Usage:                                                                                                                 breeze release-management prepare-provider-documentation                                                               -[OPTIONS] [airbyte | akeyless | alibaba | amazon | apache.cassandra | apache.drill | apache.druid | apache.flink |     -apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  -apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | -cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql |          -databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp -| git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins | keycloak |        -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres |     -presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |    -sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb |          -zendesk]...                                                                                                            +[OPTIONS] [airbyte | akeyless | alibaba | amazon | anthropic | apache.cassandra | apache.drill | apache.druid |        +apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy +| apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | +clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |        +common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab |   +facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins |    +keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc |       +openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone |       +postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex |    +ydb | zendesk]...                                                                                                      Prepare CHANGELOG, README and COMMITS information for providers. diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt index 9ada882dbcbf1..a54080b6bfbb5 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -0a45b898de7677eed8aa19c7e4f07345 +c33a2f6d00a3a8dbec8b56c1c2d88d54 diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.svg b/dev/breeze/doc/images/output_release-management_publish-docs.svg index b01450b3c1f6b..6c08dc6c566d0 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.svg +++ b/dev/breeze/doc/images/output_release-management_publish-docs.svg @@ -188,7 +188,7 @@ Usage:                                                                                                                 breeze release-management publish-docs                                                                                 -[OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | apache-airflow | apache-airflow-ctl |               +[OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | anthropic | apache-airflow | apache-airflow-ctl |   apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.txt b/dev/breeze/doc/images/output_release-management_publish-docs.txt index ca7d9b2eebe9f..9c2563d46b87b 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.txt +++ b/dev/breeze/doc/images/output_release-management_publish-docs.txt @@ -1 +1 @@ -30b0621c66436e215e0d34ec9bfd407b +71cd889146297a3b78c71773b6bb5bcd diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg index 71bce1509344e..a9d2e173ef2b8 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg @@ -182,8 +182,8 @@ ╭─ Generate provider requirements flags ───────────────────────────────────────────────────────────────────────────────╮ │--python-versions Comma separate list of Python versions to update sbom from (defaults to all historical python   │ │versions) (3.6 | 3.7 | 3.8 | 3.9 | 3.10 | 3.11 | 3.12 | 3.13 | 3.14)│ -│--provider-id     Provider id to generate the requirements for (airbyte | akeyless | alibaba | amazon | │ -│apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | │ +│--provider-id     Provider id to generate the requirements for (airbyte | akeyless | alibaba | amazon | anthropic │ +│| apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | │ │apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | │ │apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | │ │atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | │ diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt index ff491c1b8cf35..bb3262922cec4 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt @@ -1 +1 @@ -4e0a8dab83f1d257bce1cb90b778b009 +84f322f2ffa9ded046c2d3678135a8f7 diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg index 48033b138d1af..ff1db47a5f239 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg @@ -203,7 +203,7 @@ Usage:                                                                                                                 breeze workflow-run publish-docs                                                                                       -[OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | apache-airflow | apache-airflow-ctl |               +[OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | anthropic | apache-airflow | apache-airflow-ctl |   apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt index bcdbd541efa08..01d247c731835 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt @@ -1 +1 @@ -01f2b301d8965951b49e2c8cd114700a +6a419bdaebbe36070dfea872c87a2a23 diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 14b5752784dd4..1fa32b2a28f5e 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -817,7 +817,7 @@ def get_airflow_extras(): { "python-version": "3.10", "airflow-version": "2.11.1", - "remove-providers": "common.messaging edge3 fab git keycloak informatica common.ai opensearch", + "remove-providers": "anthropic common.messaging edge3 fab git keycloak informatica common.ai opensearch", "run-unit-tests": "true", }, { diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 54930c2a04c05..a607704fbf9f2 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -937,6 +937,7 @@ KerberosClient keycloak Keyfile keyfile +keyless KeyManagementServiceClient keyring keyspace @@ -1033,6 +1034,7 @@ maxcompute Maxime MaxRuntimeInSeconds mb +MCP md mem memcached @@ -1041,6 +1043,7 @@ memorystore memray Mesos mesos +MessageBatch metaclass metadatabase metadataStores diff --git a/providers/anthropic/.gitignore b/providers/anthropic/.gitignore new file mode 100644 index 0000000000000..bff2d7629604d --- /dev/null +++ b/providers/anthropic/.gitignore @@ -0,0 +1 @@ +*.iml diff --git a/providers/anthropic/LICENSE b/providers/anthropic/LICENSE new file mode 100644 index 0000000000000..11069edd79019 --- /dev/null +++ b/providers/anthropic/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/providers/anthropic/NOTICE b/providers/anthropic/NOTICE new file mode 100644 index 0000000000000..a51bd9390d030 --- /dev/null +++ b/providers/anthropic/NOTICE @@ -0,0 +1,5 @@ +Apache Airflow +Copyright 2016-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/providers/anthropic/README.rst b/providers/anthropic/README.rst new file mode 100644 index 0000000000000..a42bbbd2c662b --- /dev/null +++ b/providers/anthropic/README.rst @@ -0,0 +1,83 @@ + +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + ``PROVIDER_README_TEMPLATE.rst.jinja2`` IN the ``dev/breeze/src/airflow_breeze/templates`` DIRECTORY + +Package ``apache-airflow-providers-anthropic`` + +Release: ``0.1.0`` + + +`Anthropic `__ provider for Apache Airflow. +Wraps the official Anthropic Python SDK to run the Claude Message Batches API +asynchronously from Airflow, plus direct message and token-counting helpers. + + +Provider package +---------------- + +This is a provider package for ``anthropic`` provider. All classes for this provider package +are in ``airflow.providers.anthropic`` python package. + +You can find package information and changelog for the provider +in the `documentation `_. + +Installation +------------ + +You can install this package on top of an existing Airflow installation (see ``Requirements`` below +for the minimum Airflow version supported) via +``pip install apache-airflow-providers-anthropic`` + +The package supports the following python versions: 3.10,3.11,3.12,3.13,3.14 + +Requirements +------------ + +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.12.0`` +``anthropic`` ``>=0.101.0`` +========================================== ================== + +Cross provider package dependencies +----------------------------------- + +Those are dependencies that might be needed in order to use all the features of the package. +You need to install the specified providers in order to use them. + +You can install such cross-provider dependencies when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-anthropic[common.compat] + + +================================================================================================================== ================= +Dependent package Extra +================================================================================================================== ================= +`apache-airflow-providers-common-compat `_ ``common.compat`` +================================================================================================================== ================= + +The changelog for the provider package can be found in the +`changelog `_. diff --git a/providers/anthropic/docs/.latest-doc-only-change.txt b/providers/anthropic/docs/.latest-doc-only-change.txt new file mode 100644 index 0000000000000..2c1ab461a9c8e --- /dev/null +++ b/providers/anthropic/docs/.latest-doc-only-change.txt @@ -0,0 +1 @@ +da9caffdbbeab1917e1cec5726e50af5f14a5206 diff --git a/providers/anthropic/docs/changelog.rst b/providers/anthropic/docs/changelog.rst new file mode 100644 index 0000000000000..57383492e8320 --- /dev/null +++ b/providers/anthropic/docs/changelog.rst @@ -0,0 +1,26 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +``apache-airflow-providers-anthropic`` + +Changelog +--------- + +0.1.0 +..... + +Initial version of the provider. diff --git a/providers/anthropic/docs/commits.rst b/providers/anthropic/docs/commits.rst new file mode 100644 index 0000000000000..2c44b2b758579 --- /dev/null +++ b/providers/anthropic/docs/commits.rst @@ -0,0 +1,34 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + + .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + `PROVIDER_COMMITS_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN! + +Package apache-airflow-providers-anthropic +------------------------------------------------------ + +`Anthropic `__ + + +This is detailed commit list of changes for versions provider package: ``anthropic``. +For high-level changelog, see :doc:`package information including changelog `. + +.. airflow-providers-commits:: diff --git a/providers/anthropic/docs/conf.py b/providers/anthropic/docs/conf.py new file mode 100644 index 0000000000000..e1ccf1ad0f465 --- /dev/null +++ b/providers/anthropic/docs/conf.py @@ -0,0 +1,27 @@ +# Disable Flake8 because of all the sphinx imports +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Configuration of Providers docs building.""" + +from __future__ import annotations + +import os + +os.environ["AIRFLOW_PACKAGE_NAME"] = "apache-airflow-providers-anthropic" + +from docs.provider_conf import * # noqa: F403 diff --git a/providers/anthropic/docs/connections.rst b/providers/anthropic/docs/connections.rst new file mode 100644 index 0000000000000..26937763900cf --- /dev/null +++ b/providers/anthropic/docs/connections.rst @@ -0,0 +1,106 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/connection:anthropic: + +Anthropic Connection +==================== + +The `Anthropic `__ connection type enables access to the Claude API +through the official Anthropic Python SDK. + +Default Connection IDs +---------------------- + +The Anthropic hook points to the ``anthropic_default`` connection by default. + +Configuring the Connection +-------------------------- + +API Key + Your Anthropic API key. Required for the first-party API (``platform="anthropic"``), + and also used as the Microsoft Foundry API key (``platform="foundry"``, together with + the ``resource`` extra). Not required for the ``bedrock``, ``vertex`` or ``aws`` + platforms, which authenticate through the respective cloud provider's credential chain. + +Base URL (optional) + A custom base URL for the first-party API (for example, an LLM gateway or proxy). + +Extra (optional) + A JSON dictionary of additional parameters. All keys are optional: + + * ``platform`` — which client to build: ``anthropic`` (default), ``bedrock``, ``vertex``, + ``aws`` or ``foundry``. + * ``model`` — default model id used whenever an operator or hook call does not pass + ``model`` (e.g. ``hook.create_message(...)``, ``hook.create_agent(...)``). Set it here + to change the model without editing Dags; falls back to the provider default + (``claude-opus-4-8``). + * ``aws_region`` — AWS region for the ``bedrock`` platform. + * ``project_id`` / ``region`` — GCP project and region for the ``vertex`` platform. + * ``resource`` — Azure resource name for the ``foundry`` platform. + * ``anthropic_client_kwargs`` — a nested dictionary forwarded verbatim to the client + constructor (for example ``timeout``, ``max_retries`` or ``default_headers``). + + For example, to set the client timeout: + + .. code-block:: json + + { + "anthropic_client_kwargs": { + "timeout": 30, + "max_retries": 5 + } + } + +Workload Identity Federation (keyless auth) +------------------------------------------- + +For the first-party ``anthropic`` platform you can authenticate with short-lived OIDC +tokens instead of a static API key, via `Workload Identity Federation +`__. +Two ways: + +* **Configured on the connection** — leave the API Key empty and set a ``workload_identity`` + block in ``extra``: + + .. code-block:: json + + { + "workload_identity": { + "identity_token_file": "/var/run/secrets/anthropic.com/token", + "federation_rule_id": "fdrl_...", + "organization_id": "00000000-0000-0000-0000-000000000000", + "service_account_id": "svac_...", + "workspace_id": "wrkspc_..." + } + } + +* **From the environment** — leave both the API Key and ``extra`` empty; the SDK resolves + credentials from the standard ``ANTHROPIC_FEDERATION_RULE_ID`` / ``ANTHROPIC_ORGANIZATION_ID`` + / ``ANTHROPIC_SERVICE_ACCOUNT_ID`` / ``ANTHROPIC_IDENTITY_TOKEN_FILE`` environment variables. + +.. note:: + + ``ANTHROPIC_API_KEY`` in the worker environment **shadows** federation — unset it where + the worker runs if you intend to use WIF. + +.. note:: + + The Message Batches API, token counting and the Models API are served only by the + first-party Anthropic API (``platform="anthropic"``) and Claude Platform on AWS + (``platform="aws"``). They are **not** available on Amazon Bedrock, Google Vertex AI + or Microsoft Foundry; the hook raises a clear error if you call them on those platforms. diff --git a/providers/anthropic/docs/index.rst b/providers/anthropic/docs/index.rst new file mode 100644 index 0000000000000..d859d1a75f574 --- /dev/null +++ b/providers/anthropic/docs/index.rst @@ -0,0 +1,130 @@ + +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +``apache-airflow-providers-anthropic`` +====================================== + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Basics + + Home + Changelog + Security + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Guides + + Connection types + Operators + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Resources + + Python API <_api/airflow/providers/anthropic/index> + PyPI Repository + Installing from sources + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: System tests + + System Tests <_api/tests/system/anthropic/index> + +.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + + +apache-airflow-providers-anthropic package +------------------------------------------------------ + +`Anthropic `__ provider for Apache Airflow. +Wraps the official Anthropic Python SDK to run the Claude Message Batches API +asynchronously from Airflow, plus direct message and token-counting helpers. + + +Release: 0.1.0 + +Provider package +---------------- + +This package is for the ``anthropic`` provider. +All classes for this package are included in the ``airflow.providers.anthropic`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-anthropic``. +For the minimum Airflow version supported, see ``Requirements`` below. + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider distribution is ``3.0.0``. + +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.12.0`` +``anthropic`` ``>=0.101.0`` +========================================== ================== + +Cross provider package dependencies +----------------------------------- + +Those are dependencies that might be needed in order to use all the features of the package. +You need to install the specified provider distributions in order to use them. + +You can install such cross-provider dependencies when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-anthropic[common.compat] + + +================================================================================================================== ================= +Dependent package Extra +================================================================================================================== ================= +`apache-airflow-providers-common-compat `_ ``common.compat`` +================================================================================================================== ================= + +Downloading official packages +----------------------------- + +You can download officially released packages and verify their checksums and signatures from the +`Official Apache Download site `_ + +* `The apache-airflow-providers-anthropic 0.1.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-anthropic 0.1.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/anthropic/docs/installing-providers-from-sources.rst b/providers/anthropic/docs/installing-providers-from-sources.rst new file mode 100644 index 0000000000000..a72b45ffaa6e8 --- /dev/null +++ b/providers/anthropic/docs/installing-providers-from-sources.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../devel-common/src/sphinx_exts/includes/installing-providers-from-sources.rst diff --git a/providers/anthropic/docs/integration-logos/Anthropic.png b/providers/anthropic/docs/integration-logos/Anthropic.png new file mode 100644 index 0000000000000000000000000000000000000000..2c66ccf7963fb55b8a0057107d0da94b401594dc GIT binary patch literal 11562 zcmXAv1yob-|Htn}cXy3$7@fjsko?jiFhD{=Ksp`WjdUs94N5nNh@^y+fCxyp&_F^4Z=jwK*G zin;GklsmV0p$&|kDA3f_=4Zuu!xy43%gM78I-uayf3a`9CsKcPevPk%y=FSV?^DfF z+Vm#d1Q7oLfu)IUnQ04t_(I%u;7{snU2^*mJ%z-lg`QcyNtEjzN?Tr$Xtn})A_dPm zhgSg^n*qUV^uLs>#IdXHzXbgllOz4s8g(~#0foy&#*w$<#izws@G$vi)GJ@w&y9(n zhG@Y$%ZFxvvUjzgSHD>ZFO8uSjmVtc?t*>SH=@9!Te`qT#3(PODcUCzj2co~eZiJo zr@jWn1qMfDNa9^HRkrF)$c%cD&jXUJm}R=D&|^>l)N>K#B|#G887}24Vb4#|1>;bF zNYw!aCxrq=!=ViA?G~7BNY!kXLtVdJd0D7XB6P9rXURzPmuQLj?o9n`@qr~KCcBaJ zr0p7*=vPVma8zE54d`ME+%VXxL34}oMcpmc2-)YHs^e@z&9&5K@GE{keBu+bvwqVB_L*x_&kZL5xkHL=sKfjViG%FQ5rnR&JdR<_&N14TgU9 zcw-2`brhVH390|7!fy*m^+~eT}06WPJ{+ky+pOlI)%!_Gp6cOHJO;HRuvv$S(u7yD@5zH;q*kggU;V zSvI*D3<3xXgnpn2@zqsldn1;Dn?hm;1O^hBK0=atKD&g#WSrVC!00JFx0V0)$zjmI z%UZ3&nw9{mJ2VO2WL{f_UUYc*QZT0HWzN2%IlD^aP(M^2LW9G3vXnLpmJn(Y{Mv^@ z{!!9-cdX2V_au#KhMy4qI}OqJqr4UlDu8)kqEu&RvdvD?q1}M_f-oBUjRnMPyloX{ zgBRyT{-Vy7#1dGxdnTKvOiDIhE&{IuJI@=VC{sv*d$y8#os8!%Sr(6PhP5cd=QeNW4U5@gXMnP>}ugoz=h{Pi|6G^rLMi z+WX7rll+YxgtkiOpvS_`uW4Drd0Ux6aX-qn|BY^E+?6sM+nhH+KC_gTtpAaVO4P(} zC&0!P2o71A;LMsITNs5svi5@gJ8Wgf0HHRzQ#O<}9M8 zn5e*upUWyBX{)AOtChUAF2is<@&2Yi(d}!;ouIx4U|d*QYocYM4&2s7j0uS6d6EHo z3pKbCc1feVy6Y2^4MR+QXCyT*r^Irj-L= zd5A#cs*z&i(>1!<-yox~zhcUZ)*JiV(uKbwEd1CB0XG+&V2lkHy?ny2qVtb3M@0by z*1w2cgBteVu=UKt(!im|%hErJZL`Y;U(wGr61_7T7{ZSk3iKD;eYVDPcXvBmXt(LP zNyluxm!v=CBOtJfzEw}O(iu0TiT-0k)dCw8SV6!QwLisV=G62JIEQ$88yIuYtxqLI8v=*B~GR{UQ^>zfOGn6ViNu0+W*R#0f{sy+VJ5 zd7EhKhAY`w`x0$>z2=~&@EV*=_&gB3jGDxp_G9y>iMj4*`1ejZSB8~=GS868>pT2+ zsf7qo_K3+@i&8LffPtKVM#5JR$_|o;+fc~8U0I(lz@!Zm+wgjV-qY0J^Y-( zRLclXs);~^v(`6o(e=njBg`{MkX7@~I|8JFR+2Y@`xX+~@(70>c=5<`q$d4(~9F2s;Qu3QZ}2jYW=hSl=e`+B>Hqt z@h1C(3MmWDXgd|o_y0b2q`Gh%ls-1@ZhWaQUDRX-WEl{N8@solCx}(s#|&2|NRpe2 zxhhK_qO9(xJ_K8pjdGHv8s5#bCTqTrQ&%sN?hj}7QtB5s#qP?+y)D~oN<*`eziuTw zl|GlABEx&kcC>4?x6wNn;3vX0v8cuKixdw(6Tev7bz(M9jKzBEd#jQOTh7UK>WloE zLy5klg1lr+yKleTBAggl;iYOj>SP=XRJJw6R&+^o&M?MRc4OS+=4^TZDp${ocfd3v zVZ)52_q+chqQ_G4gEn4lF%D>BWUjzVoAtMb>=gV*KtEkjod%~Y>!$ktZ~wV>8CU%q z|49*qB|fqD4|kxYUfk7Tj9s(`@I#57hR6~YmNsY*S|OMjJ989-K@?iQYC90eq>!v)%6$(7>s+<!6jdA6$EoNgrdY^f`7au6LKT=bo@97>s z)%K_Mx>KwJHI+}zHC*jWx>nv0trl8Wl537v;?LYcEOino*o9N^q4y?YKx|(`zUEm% znA#nhRrlL{9Y(8fWqFMzm{<8!rrB*PJ_MdvFXcI=(feX$)MXv{IZfW-`X&i1pJdDs zyd`M1yCMp>9MLFwz}uWMMi~G?#*ff+D(BnrO&Q%@|NSsDDtfbqfua>xLp3r0lRzx& zqpUf_w*=TZ*yb2#8CdcGL4SPEo$}GvEtdI2=6{uk_E-|bwZ#Gk-W|CGw(|A_$hE^T zU)q+(>w;fvqIVN?J-ZI=SElk(GLpv05~+YmG1trO1J;8pVVOV5rp_J-MY`m&MeTI}ZuZiGmEL63s|fLu?rA__tF-tQcP)Kg zsvtUZus|?+e%Q7Y#jy$s?<4qNLcWR@u=AQhiN$aMqpB(%x|7Ie2=ZOQG`oXctiJ3M zGC8!#Uqx-T#cbATeOe?u!L=XQ0(e}ju{UhkxBZaMakf;eF-iy8n}4RUv@gl(jo6?J z zTB)bz!@vU-F54NgtV8;|@>qc_UwsyqNYCAu>nD zVf%K~&w(zBoW0~S43Y|Z5~qu*o4u$jR-W+uy!4#kUmy8fzpKmd)JSrOqLyg`=MN3` z$up9lc{7cd-YeonVuu6X5jJ1aTech!nuS(t_LLJMQ7yFhNnYpQWEJB;?@9SX)dIN4 zZw%Thg>b*N?>?-(b;k6K6x!YbA{{T@N99kPje2V(j+5&po^!eS-30lRTsgDp6HPfp zCOtxSX6c^a{qPE@I<5$66YBp~6X`L^_Rv$q_Y^5WaCZJ>DzQzg z!{uE~J6(K6cJa08xGZmgYrv7!-uuNK<(f3EqDko_*e~|=+ID2gI?h7qT}3Ax)8}K zRz??)c4KWPi6)u=LfQQhTYR}%vVL9bT*$%4-I)(+p8W0V_|fsXn&o=UwKm;zM5G=0 z-0w%5L+5WCYQ`XL)9|bC6PeOsjZQlj$Kjdu&{zlO1P%{+42WInI)70Z?7rRENkejf z!Rq^Nef{SqtIVs*uS{Ii`gZC;d2n|Q7NiZA11x8)jo536&t6>fa%-40<;47w5c~B$ z%DaR4bPwPIH~esXy6GAz+5hZJ-Tg^)EC7_HccvDoU^&RiDuF*`*L;oVKUmN~4d@9k z&EeP_-MDSynFanCYFq0%QoRRZ3J}3YNGQ&Wj-}#;dhfXZlq$+wg{rZZdnPxVTrA(1 zK6jBY9!d<#FvdHVE*D}f87{D(^qnC~ShtIEWo~HoYYmu=z1?hTO3S|SY1kTwcJmdr z?LKn5C+cef35Tq$y~K4eu|DKTlDX(TvRo8@d+WeCGzfkBdQ=8_Vp*zbAw9*9B4icd zb1m@x&XjAQbpZ;+W1Ek<9{g?FeT-^ty4#QVI6~W7jxXbQO6Hn|ZtwAQ8BQLfDeds__eq!@Y}eo`&p5Hkx-sfU^NBTmthsDU&oILXAbuwfOghTeZj+b5fhO4 za%23`E!67%<7Y95DER0TzfH6JA&Lx^fEjuotb@anc721d_J%FGPQ*{w#qwSp@6^=1 z&qM#3A3OSwm3%cbxXe>K%4()*=)>pFh+_(k8*X6G@ycL|k9NlCGMPq*ssZWPup>&O zXw9&d=&?>@T`=Fv!)l(R1sWiD`lwA9OaRBaMa()SjQXfVSSr`PB zX;cfQlz(AvG&{f_aCjpR>7!_6s~NL#_REFxEt+rp)~y{&d5T3!fGF2p^8N-`er*2D zTZA=Y$GOZmmKmhtb5OxDi-$5tcw#6-81c(o+?P9Sxs?ES-1&t&DV^b`lm6Pz2p@_X zn%bHuVaPX66=H9?Z@JGWyW~0)fKH}Usr+OWfs^>*H39)OS^$Q>wC0ub4hM-4XwD31 zVTnC*9fJVtLyvm&4LLA)7X#0!870Au?qr0Shhti97}U*Gi6uB@)!o4gDFhYP?1`a8 zR!`hNt~|>GaFAi7h&ilPGo^jW&?o;lv|)6wB-S0r+AzLnO5>j=AZO+4w+0IsH5FmO z^}&>3-`-yPajWuhzkc<}Yohk;ntch}0=p#cwrD{i{s$2F`N;Sz-V!MVmP>Ss6i!vu zagj+YzjRd%i7OOv=}=U-lON>fwAgx*XlcFpRRgCkhr7rzfttdAez}=xf10q#W~mJ6 zpJC*si$Tb$nPC)GS6RnL9SpPi#I&a>C!U(kM(9S0qSE(Kz`YjEV?;Vy&yCdlhb{_L z3cQdihUXhv2w{;l?kTME;UkB`2fNOH8G%gR@Oc@QqIhB!7u5@_q6z19pl_(R8gs$I z#_wIWdAf>Yp>xG)*GTGXn?4&K=}(v_t{oT#)XOV(f2XJ>2fg3!sqPh}p~h`d1(G73 zBQS*YMv0?9Z8xa}S(B(&VXTI(q-$){R+3-n683V2T30$(bFs%aKA(sqpf$|m$SeBp}&|qiH$rh#*LP8w7;Og z$aHadC2%pAJDCsLHLb=W6ReVGNvdsotJA&i!-1*|_7zl<#1D@FZk-dwiWu+v=@Knl z*O3f>+ELolniD%9O$J^jYp7^rNw3wvNU(5d z#?z&4DHRyN9+y15A*I__Glab z4OT<%qWEBQU@ZC-oP)W@mfaTSD+MZq`gTH=(PIu3yXZpOxv{-TrSc28I%OVOI=MVj z2Goo!PN-1oF;9ApV<{BXXR98A`74n#J3ZPJAaNJ#!Z0{92f-%X5@P`h8}(%x}qj zP`XHp*bv^}t~7+Qop4A435-O@8@mWA)CcUq6s^4+aXvU6LGW5c@c)M)^Nb3bC_3{R zv>J%HL56PON!Tr3*Rtu-{i`n>`{w(${6n*rDaaZ9rWevk)fd3iXo7hBg6~W}mm2)^ zFTaYQiy<+aqM6!hF%UGHmyC0vK+@+5w8T56 zu*iJ04N96V&g9t>LvS}sNM`o7I$0^n+W2K%TtI1#Fw$lwCp!V@%cR;ND~!K0)J+%J zbCF=#Bg^@d=rSJZAg7EVR$EDT$kJsqzcb zs<#9CNdu%4eMZUgP!NJ@VJ2?wrXgXphl4<0B^*<#LUrIL>h{!Rc_cO_yHIRa2j8u4 zMe@PR4@}TVgbp@boFoCV^Am_9frd&cW}nNpO+sXy9X#=sSJ45u%B4(}P0w5xQMf-1}%uv%ba2K76P*;WR$l8Hrqrf_#-lY9nDS zO65@ngfKyxL2}T~__v&zGH-Ou+p>GnLMp-)7*m`pr7{u2 z@WA;aUu=gD`7y@`=TALMx0&%a!#Lpom`_mS3Hly7N;LlbHt8{+fb(tqC6JnWx}V+>$&kiyxG>jD<6c0hdi z$oL?vkc*GTb1iEshj zubo@2H2V)jRj1#6_1Ca=OU(+d;hAiBXN`N70<(~OsC778IorCJ4KqVe3x#O6t zB!y7g6a0i&4*&9g!<-~2`)k~u;NJ(V_e1vn@xm`eN)_>`-$-X^DUH(D5x?2=Og>uX zM@_}`Pr`6hAGH+H^D|R@q1z+=X!mCF0_A6VJ($wINxRukDP5vGJ&Jz$Rk|F0#L(@| zwl7P(1vT@j-DZ?ASV;NB>X3jsCJn3JjP9A!t$=CLUr^T7lrH! z;Cb(*A+qocdxJLZ!8DePX>W4m@2pDs>>Fata_qlWk?et2-@Y5i(e!B|p0-;;= z-s2%FXH1WiPh8>u&P(8sH^#$;BAsob0lonec9tnWF6@<{J^Q1GMEqWI-`w%Z)hWdX z!9G&J#1Zc^$-R0>481Z z?-#frLf}Cfy;1)EzJz4(_(X5$h^6v8;cDS)gSx{% z+ja_lLehAr)Cg?Nosf6GY`1!q-Y09IRZrTMsdx@M5n!Vj87OZ0EYd|&Z4c#L=r`=w z%o(4u39Vx}4K+rDc&z+f91e;J135;0q*)%G-bHj(RGTrXOPZ;4d2d2CuF`wCoP8j6 z&?1~N-QZ(i@@T;=4DwGep-0xj-wx}uWRW<4X~#oE%DU&JbIT7-WL&ImLkDY+s-kxl zx2FerzPgU>dSTzk*SH^MkV!bJ3k>qS$qp?k{6sPOIKPfT<0#Fp&S`-di8lZrxhZYR z9^d->vvJ`(Yjp#CcW~#yzLlvbKb_pX_ud@)Fxx)fM$ySa8A-Al|InFrsss>mLrPZ}KbY{xf-$P!IvicH&T>A+0!C`1R!?^9DDh z>xHGB$ma^m+7ZzZXKeEqIcG)@a`*8{%oVxW9J|XLN&Ez(Yr5wf|MWtgkv{TDy^B{h z4yTh{f!wKjk?KXSuSQ=njI?9B?%($1W-*+8k zW8Fe-_CJ+$b$T5f7X$u21mhR@V#7Vq7`=S7j)Z~UN-~%mIti1kM3&nR=iQsX{aWvrlMyyH?dSLQ;w>mK(ce?9a+(zM5cZ0g~#H0*Se>i8*2Tz z8tBP?-|F8XLlWv@MP5b!#GdS%-(-FNN*WT_1QPPjyjlQF<$p$qIl$sGtg>Oq{PAv3 z`6!T#u9bSnQiu3ayWw2bNW6__aE6T;nU1jq96eDR9gVj1>KXk20C)D^T7VN8p&)t5 z`FuE$M}6@$DTX;tce%Eu?_ci)T^s-t?jtT*7#$2Q(oC`f{+V1|`aU1GRRMDdt`663 zwPA_z*Nv@-D-{?sk@*h(=v!m@pxw8oho1ER$pDA4#otOcaBiPr!ryB_%N%w)340_r^(Ny>Jf=mis+WpmRur0!?+Zo_0n&*ub$Oipd#(L#V>vEoeT44 zPcue5RY4}IdAN{j=fxiQ$i#c*+(%PnZO}JLXS2zH?f=%2UH+;69T1=8VOYIMj{fGM zv}u6s4Xl?fEHPM1rTcB6+yKV?5hpagbEs&RsZ0xmsVm>WI_{j*0=vML9MFjS@Z3`uoQ{OX1YK(G=gPHywbYS=^}9hec0lLd`0Ol!(1T^rloVt~F2^g@c7-_Z z1YaosAaTwj^OaqMA@%)Q+LVB}3OBJvP+98BP)rg7nUq!-P$Xh5$k3zhthGJ4e~+!%EA}XgLh$Z;Z-$1TaN_V1d3vNCx?(S zse?K%F(rrw>CC0{Z1s@%2w5$jT{GwW!1;TjJeSlh0DsnM=Q3YP+2OgE_U$viihBhn zxvGV0vY%v6s9Suk-)xH(aE1yz%gV!H!gPq8Gcz`9maw%S;-pQijo(Sj$$lD#$LO<{ z3N%E;gdS;*SJpXnO~x;Fhty9|3wCqUnQ=NBR2`;jW>6^ z^bc92IdbFC3cYYyqmtDYsSkLmx4WtPRaDqIhYKmMs=v8KaeQfrp3q{Yb=9ADnC^#J z5MVCs3(9-O6g;*l?w`-RoqL-*6+UqVs=f8O0J}q3f4saS!+(3U~Zy6 zi-EITC3|$odR>*$#&=QXYhEC<^M3JapS3#R-zK~)LP;u)twjy2A8*HBK z8*qdcrcUQ_+}B6siB$aM?Wy$jI~%qYl*7!XORcfvi-S5>^tOc(Gc*+aIi8~s8TXb3 zCeQ1@DyQn|a8q|6;368O;aI<#9qD!j?EILD{DN(is4xDZ-t>{*&33&+stQw5nIQ8_ z?Fr9PBHpL`)H&Cwfb?a@;X1wApv`Eu)Uf647oHnt*Q7@lJW|cdf^rB$}TrMs!SXy><}R18JVVPj!zB{QZ*0l50IIk*7U2_T0Q)7w2ilS z-(1%ki@|lM{KLujuEu{U2hWd}&|c_4`Iy5F-SXI!G+?zN3*UwQ8ORjnhEK!T z@iMpiT})$&q8^gHTG-#|8BmdGI6r4T?jlp=+Q)#-|Ia3K~f*aH#ej^iZDc^>~^H4872H)IGj+`ZtjS&va{c8rGucUhNgEsnj zrJ*+0hei?I7CM3v%jSSk?YXUb{xdB|u3f&6JWRT*IS!j-i1i%~gYbR5DCP1Rr0MuI zn?Dstx#=^i-rZB}7t=QclvxqMvAvkSCiT5*W85K7m<#LhH9%QDiZ7sIp0cQi9&su0 zEo#yYEN=Rx`S6zOQczGQ(-fv+=%&Q-XvX-fJwzrmyRa$Ee(!Rg2APwPg~bcdoPC9y z_=Sld`;!s=Ze#XQ`tmSU#qJ~JT{3L{!3>4 zWx@Op-`C7zG$F#M-S$FFU+Ir}Ory8o=JYQ!hP(WrEjA@z&Ric==@>Jr3Sk;X5%uqmD81fmgF`3 zW80jUimsJAG(Lf0y&e>Iu&KHa3i0uy*l)BU@OR*!<=!Ir*T^Lfuco43UJ@3WK%7rm zt=QPYi3>x1p-n-3!b6_nH`wBV6PSLZ$~%5s(&BHXz7Ln zQKKRQZ75&-32pIQ;zC~jk=ubgatCvaFZwMhcC91pC~0>-W&#Yh@94~ir*RF%#*SbS zGLL?=LlRDmX-&by3Tf+~u^{#f=dsVYU--mHiSc;MDCWCK&T zGqo6fnc4>#6Q-w9G&QH(G30qBy^%{UoZ!Z2`bFhGHW2U6>*o-}7yzym{I{5~BLxOf zlD{M3#4#l+(T?$%wd6znD>{I%oeb5Q87(9WGo=nk9&o5OzPQ0x2<0q_3597Wtz(f> zGT@~TMn1gz_qD71k9$*(PF3F;DXKLSi^PXNe!1AC*nx6#5Vn7~h*aRQ_|or{LZwSc zHz($hA|@PNNkmo_K}>dSgbfzRHZR1F3pwd2i^>)%e?tn*M;_B|v&r_5Ma^pq1^viN zWm7=Nnk)GkT8J_1FWWEC%>%Im)EfQq?O%2y`VgrzCJJt&3IhFn?IPPem#@&(&%uFcK)}j6W0pjnZF#`__ from Airflow. +Message Batches process many ``messages.create`` requests asynchronously at 50% of +standard cost; most complete within an hour, with a 24-hour SLA — a good fit for +Airflow's deferrable execution model. + +.. note:: + + For interactive, single-call or agentic LLM workloads, prefer the vendor-agnostic + ``apache-airflow-providers-common-ai`` provider with ``model="anthropic:claude-opus-4-8"``. + This provider focuses on the batch/async surface and direct SDK access that the agent + abstraction does not model. + +.. _howto/operator:AnthropicBatchOperator: + +AnthropicBatchOperator +---------------------- + +:class:`~airflow.providers.anthropic.operators.anthropic.AnthropicBatchOperator` submits a +Message Batch and waits for it to reach the terminal ``ended`` status. In deferrable mode it +releases the worker slot while an +:class:`~airflow.providers.anthropic.triggers.anthropic.AnthropicBatchTrigger` polls for +completion. + +The operator returns the **batch ID only**. Pull the per-request results with +:meth:`~airflow.providers.anthropic.hooks.anthropic.AnthropicHook.stream_batch_results` and +persist them to object storage — results can be very large and must not be pushed to XCom. +Results are retained for 29 days after the batch is created. + +Parameters +"""""""""" + +* ``requests`` — a list of ``{"custom_id": str, "params": {...}}`` dicts, where ``params`` is a + ``messages.create`` payload (``model``, ``max_tokens``, ``messages``, ...). +* ``conn_id`` — the Anthropic connection ID (default ``anthropic_default``). +* ``deferrable`` — run in deferrable mode (defaults to the ``operators.default_deferrable`` config). +* ``poll_interval`` — seconds between status checks, in both the synchronous and deferrable paths. +* ``timeout`` — seconds to wait for a terminal status; defaults to 24 hours (the batch SLA). +* ``wait_for_completion`` — if ``False``, return the batch ID immediately after submission. +* ``fail_on_partial_error`` — if ``True``, fail the task when any request errored or expired. + Defaults to ``False`` (succeed and log a warning so successful results are not discarded). + +.. warning:: + + A task retry re-submits a **new** batch. Prefer ``retries=0`` on this task. The submitted + ``batch_id`` is pushed to XCom under key ``batch_id`` immediately after submission, so a + crashed run never loses track of an in-flight batch. + +Example +""""""" + +.. exampleinclude:: /../tests/system/anthropic/example_anthropic_batch.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_anthropic_batch] + :end-before: [END howto_operator_anthropic_batch] + +.. _howto/sensor:AnthropicBatchSensor: + +AnthropicBatchSensor +-------------------- + +:class:`~airflow.providers.anthropic.sensors.anthropic.AnthropicBatchSensor` waits for an +already-submitted batch (by ``batch_id``) to reach a terminal status. Pair it with +``AnthropicBatchOperator(wait_for_completion=False)`` for a fire-and-forget submit followed +by a re-entrant await — because the sensor only polls an existing batch, retrying it never +re-submits, which sidesteps the "retry creates a new batch" hazard of a waiting submit task. + +It applies the same terminal-status policy as the operator (skip on full cancellation, +``fail_on_partial_error`` to fail on errored/expired requests) and supports ``deferrable`` +mode via the shared trigger. + +.. code-block:: python + + from airflow.providers.anthropic.operators.anthropic import AnthropicBatchOperator + from airflow.providers.anthropic.sensors.anthropic import AnthropicBatchSensor + + submit = AnthropicBatchOperator( + task_id="submit", + requests=requests, + wait_for_completion=False, # fire-and-forget; recommend retries=0 + ) + wait = AnthropicBatchSensor( + task_id="wait", + batch_id="{{ ti.xcom_pull(task_ids='submit') }}", + deferrable=True, + ) + submit >> wait + +.. _howto/operator:AnthropicAgentSessionOperator: + +AnthropicAgentSessionOperator +----------------------------- + +:class:`~airflow.providers.anthropic.operators.agent.AnthropicAgentSessionOperator` runs a +`Managed Agents `__ session: +Anthropic runs the agent loop server-side while the worker drives a session and waits for it +to finish. Unlike the ``common.ai`` provider (a *local* pydantic-ai loop), the loop and its +tool-execution sandbox run on Anthropic's infrastructure; the worker only orchestrates. + +**Agents and environments are created once** (via +:meth:`~airflow.providers.anthropic.hooks.anthropic.AnthropicHook.create_agent` / +:meth:`~airflow.providers.anthropic.hooks.anthropic.AnthropicHook.create_environment`, the +``ant`` CLI, or the Console) and referenced by ID on every run — the operator never creates +an agent per task. Configure the agent for **autonomous** operation (no client-side custom +tools or ``always_ask`` permission) so the session reaches ``idle`` (turn complete) rather +than blocking on input the operator cannot supply. + +Provide exactly one of ``message`` (a single user turn) or ``outcome`` (a +``user.define_outcome`` rubric the agent iterates against until satisfied). The operator +returns the **session ID only**; pull artifacts the agent wrote to ``/mnt/session/outputs/`` +afterwards via the Files API (``scope_id=``). + +Parameters +"""""""""" + +* ``agent_id`` / ``environment_id`` — IDs of a pre-created agent and environment. +* ``message`` — a single user message to start the session (mutually exclusive with ``outcome``). +* ``outcome`` — a ``user.define_outcome`` payload (``description`` + required ``rubric``, + optional ``max_iterations``); mutually exclusive with ``message``. +* ``conn_id`` — the Anthropic connection ID (default ``anthropic_default``). +* ``deferrable`` — run in deferrable mode (defaults to ``operators.default_deferrable``). +* ``poll_interval`` — seconds between session status checks. +* ``timeout`` — seconds to wait for a terminal status; defaults to 24 hours. +* ``vault_ids`` — vault IDs providing MCP/credential access to the session. +* ``session_resources`` — files, GitHub repos, or memory stores to mount (forwarded to + ``sessions.create`` as ``resources``; renamed to avoid the reserved ``BaseOperator.resources``). +* ``session_kwargs`` — extra keyword arguments forwarded to ``sessions.create``. + +.. note:: + + Completion is detected accurately for both modes. A ``message`` run inspects the + terminal ``session.status_idle`` event's ``stop_reason`` (correlated against the + kickoff event): ``end_turn`` succeeds; ``requires_action`` and ``retries_exhausted`` + raise an error. An ``outcome`` run is judged from the ``outcome_evaluations`` verdict. + The agent must still be configured for autonomous operation (no client-side custom + tools / ``always_ask``). + +.. exampleinclude:: /../tests/system/anthropic/example_anthropic_agent.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_anthropic_agent_session] + :end-before: [END howto_operator_anthropic_agent_session] diff --git a/providers/anthropic/docs/security.rst b/providers/anthropic/docs/security.rst new file mode 100644 index 0000000000000..66c6f79a4ecfc --- /dev/null +++ b/providers/anthropic/docs/security.rst @@ -0,0 +1,38 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Releasing security patches +-------------------------- + +Airflow providers are released independently from Airflow itself and the information about vulnerabilities +is published separately. You can upgrade providers independently from Airflow itself, following the +instructions found in :doc:`apache-airflow:installation/installing-from-pypi`. + +When we release Provider version, the development is always done from the ``main`` branch where we prepare +the next version. The provider uses strict `SemVer `_ versioning policy. Depending on +the scope of the change, Provider will get ''MAJOR'' version upgrade when there are +breaking changes, ``MINOR`` version upgrade when there are new features or ``PATCHLEVEL`` version upgrade +when there are only bug fixes (including security bugfixes) - and this is the only version that receives +security fixes by default, so you should upgrade to latest version of the provider if you want to receive +all released security fixes. + +The only exception to that rule is when we have a critical security fix and good reason to provide an +out-of-band release for the provider, in which case stakeholders in the provider might decide to cherry-pick +and prepare a branch for an older version of the provider following the +`mixed governance model `_ +and requires interested parties to cherry-pick and test the fixes. diff --git a/providers/anthropic/provider.yaml b/providers/anthropic/provider.yaml new file mode 100644 index 0000000000000..d8f0b71feac7b --- /dev/null +++ b/providers/anthropic/provider.yaml @@ -0,0 +1,139 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +package-name: apache-airflow-providers-anthropic + +name: Anthropic + +description: | + `Anthropic `__ provider for Apache Airflow. + Wraps the official Anthropic Python SDK to run the Claude Message Batches API + asynchronously from Airflow, plus direct message and token-counting helpers. + +state: ready +lifecycle: incubation +source-date-epoch: 1780426378 + +# Note that those versions are maintained by release manager - do not update them manually +versions: + - 0.1.0 + +integrations: + - integration-name: Anthropic + external-doc-url: https://docs.claude.com/ + logo: /docs/integration-logos/Anthropic.png + how-to-guide: + - /docs/apache-airflow-providers-anthropic/operators/anthropic.rst + tags: + - ai + +hooks: + - integration-name: Anthropic + python-modules: + - airflow.providers.anthropic.hooks.anthropic + +operators: + - integration-name: Anthropic + python-modules: + - airflow.providers.anthropic.operators.anthropic + - airflow.providers.anthropic.operators.agent + +sensors: + - integration-name: Anthropic + python-modules: + - airflow.providers.anthropic.sensors.anthropic + +triggers: + - integration-name: Anthropic + python-modules: + - airflow.providers.anthropic.triggers.anthropic + - airflow.providers.anthropic.triggers.agent + +connection-types: + - hook-class-name: airflow.providers.anthropic.hooks.anthropic.AnthropicHook + hook-name: "Anthropic" + connection-type: anthropic + conn-fields: + platform: + label: Platform + schema: + type: + - string + - 'null' + enum: + - anthropic + - bedrock + - vertex + - aws + - foundry + default: anthropic + description: >- + Which client to build: anthropic (first-party API, default), bedrock (Amazon + Bedrock), vertex (Google Vertex AI), aws (Claude Platform on AWS) or foundry + (Microsoft Foundry). + model: + label: Default Model + schema: + type: + - string + - 'null' + description: >- + Default model id used whenever an operator or hook call does not pass model + (for example hook.create_message(...)). Falls back to claude-opus-4-8. + aws_region: + label: AWS Region + schema: + type: + - string + - 'null' + description: AWS region for the bedrock platform (for example us-east-1). + project_id: + label: GCP Project ID + schema: + type: + - string + - 'null' + description: Google Cloud project id for the vertex platform. + region: + label: GCP Region + schema: + type: + - string + - 'null' + description: Google Cloud region for the vertex platform (for example us-east5). + resource: + label: Azure Resource + schema: + type: + - string + - 'null' + description: Azure resource name for the foundry platform. + ui-field-behaviour: + hidden-fields: + - schema + - port + - login + relabeling: + password: API Key + host: Base URL + placeholders: + extra: >- + {"anthropic_client_kwargs": {"timeout": 30, "max_retries": 5}, + "workload_identity": {"federation_rule_id": "fdrl_...", "organization_id": "...", + "service_account_id": "svac_...", "workspace_id": "wrkspc_...", + "identity_token_file": "/var/run/secrets/anthropic.com/token"}} diff --git a/providers/anthropic/pyproject.toml b/providers/anthropic/pyproject.toml new file mode 100644 index 0000000000000..74154f78bb805 --- /dev/null +++ b/providers/anthropic/pyproject.toml @@ -0,0 +1,134 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE +# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +[build-system] +requires = ["flit_core==3.12.0"] +build-backend = "flit_core.buildapi" + +[project] +name = "apache-airflow-providers-anthropic" +version = "0.1.0" +description = "Provider package apache-airflow-providers-anthropic for Apache Airflow" +readme = "README.rst" +license = "Apache-2.0" +license-files = ['LICENSE', 'NOTICE'] +authors = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +maintainers = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +keywords = [ "airflow-provider", "anthropic", "airflow", "integration" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Framework :: Apache Airflow", + "Framework :: Apache Airflow :: Provider", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: System :: Monitoring", +] +requires-python = ">=3.10" + +# The dependencies should be modified in place in the generated file. +# Any change in the dependencies is preserved when the file is regenerated +# Make sure to run ``prek update-providers-dependencies --all-files`` +# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` +dependencies = [ + "apache-airflow>=3.0.0", + "apache-airflow-providers-common-compat>=1.12.0", + # 0.101.0 is the first release that ships AnthropicAWS, the newest of the platform + # client classes the hook imports at module top. + "anthropic>=0.101.0", +] + +# The optional dependencies should be modified in place in the generated file +# Any change in the dependencies is preserved when the file is regenerated +[project.optional-dependencies] +"bedrock" = ["anthropic[bedrock]>=0.101.0"] +"vertex" = ["anthropic[vertex]>=0.101.0"] +"aws" = ["anthropic[aws]>=0.101.0"] + +[dependency-groups] +dev = [ + "apache-airflow", + "apache-airflow-task-sdk", + "apache-airflow-devel-common", + "apache-airflow-providers-common-compat", + # Additional devel dependencies (do not remove this line and add extra development dependencies) +] + +# To build docs: +# +# uv run --group docs build-docs +# +# To enable auto-refreshing build with server: +# +# uv run --group docs build-docs --autobuild +# +# To see more options: +# +# uv run --group docs build-docs --help +# +docs = [ + "apache-airflow-devel-common[docs]" +] + +[tool.uv.sources] +# These names must match the names as defined in the pyproject.toml of the workspace items, +# *not* the workspace folder paths +apache-airflow = {workspace = true} +apache-airflow-devel-common = {workspace = true} +apache-airflow-task-sdk = {workspace = true} +apache-airflow-providers-common-sql = {workspace = true} +apache-airflow-providers-standard = {workspace = true} + +[project.urls] +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-anthropic/0.1.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-anthropic/0.1.0/changelog.html" +"Bug Tracker" = "https://github.com/apache/airflow/issues" +"Source Code" = "https://github.com/apache/airflow" +"Slack Chat" = "https://s.apache.org/airflow-slack" +"Mastodon" = "https://fosstodon.org/@airflow" +"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" + +[project.entry-points."apache_airflow_provider"] +provider_info = "airflow.providers.anthropic.get_provider_info:get_provider_info" + +[tool.flit.module] +name = "airflow.providers.anthropic" + +# Explicit sdist contents so the build does not rely on VCS information +# (flit 4.0 makes --no-use-vcs the default — see https://github.com/pypa/flit/pull/782). +[tool.flit.sdist] +include = [ + "docs/", + "provider.yaml", + "src/airflow/__init__.py", + "src/airflow/providers/__init__.py", + "tests/", +] diff --git a/providers/anthropic/src/airflow/__init__.py b/providers/anthropic/src/airflow/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/anthropic/src/airflow/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/anthropic/src/airflow/providers/__init__.py b/providers/anthropic/src/airflow/providers/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/anthropic/src/airflow/providers/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/anthropic/src/airflow/providers/anthropic/__init__.py b/providers/anthropic/src/airflow/providers/anthropic/__init__.py new file mode 100644 index 0000000000000..117734e0ae5e9 --- /dev/null +++ b/providers/anthropic/src/airflow/providers/anthropic/__init__.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE +# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES. +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +# +from __future__ import annotations + +import packaging.version + +from airflow import __version__ as airflow_version + +__all__ = ["__version__"] + +__version__ = "0.1.0" + +if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( + "3.0.0" +): + raise RuntimeError( + f"The package `apache-airflow-providers-anthropic:{__version__}` needs Apache Airflow 3.0.0+" + ) diff --git a/providers/anthropic/src/airflow/providers/anthropic/exceptions.py b/providers/anthropic/src/airflow/providers/anthropic/exceptions.py new file mode 100644 index 0000000000000..f4e81b868a185 --- /dev/null +++ b/providers/anthropic/src/airflow/providers/anthropic/exceptions.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + + +class AnthropicError(Exception): + """Base class for all Anthropic provider errors.""" + + +class AnthropicBatchJobError(AnthropicError): + """Raised when an Anthropic Message Batch fails or finishes with failed requests.""" + + +class AnthropicBatchTimeout(AnthropicError): + """Raised when an Anthropic Message Batch does not reach a terminal status in time.""" + + +class AnthropicAgentSessionError(AnthropicError): + """Raised when a Managed Agents session terminates or fails.""" + + +class AnthropicAgentSessionTimeout(AnthropicError): + """Raised when a Managed Agents session does not reach a terminal status in time.""" diff --git a/providers/anthropic/src/airflow/providers/anthropic/get_provider_info.py b/providers/anthropic/src/airflow/providers/anthropic/get_provider_info.py new file mode 100644 index 0000000000000..0a90f41b4652e --- /dev/null +++ b/providers/anthropic/src/airflow/providers/anthropic/get_provider_info.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +def get_provider_info(): + return { + "package-name": "apache-airflow-providers-anthropic", + "name": "Anthropic", + "description": "`Anthropic `__ provider for Apache Airflow.\nWraps the official Anthropic Python SDK to run the Claude Message Batches API\nasynchronously from Airflow, plus direct message and token-counting helpers.\n", + "integrations": [ + { + "integration-name": "Anthropic", + "external-doc-url": "https://docs.claude.com/", + "logo": "/docs/integration-logos/Anthropic.png", + "how-to-guide": ["/docs/apache-airflow-providers-anthropic/operators/anthropic.rst"], + "tags": ["ai"], + } + ], + "hooks": [ + { + "integration-name": "Anthropic", + "python-modules": ["airflow.providers.anthropic.hooks.anthropic"], + } + ], + "operators": [ + { + "integration-name": "Anthropic", + "python-modules": [ + "airflow.providers.anthropic.operators.anthropic", + "airflow.providers.anthropic.operators.agent", + ], + } + ], + "sensors": [ + { + "integration-name": "Anthropic", + "python-modules": ["airflow.providers.anthropic.sensors.anthropic"], + } + ], + "triggers": [ + { + "integration-name": "Anthropic", + "python-modules": [ + "airflow.providers.anthropic.triggers.anthropic", + "airflow.providers.anthropic.triggers.agent", + ], + } + ], + "connection-types": [ + { + "hook-class-name": "airflow.providers.anthropic.hooks.anthropic.AnthropicHook", + "hook-name": "Anthropic", + "connection-type": "anthropic", + "conn-fields": { + "platform": { + "label": "Platform", + "schema": { + "type": ["string", "null"], + "enum": ["anthropic", "bedrock", "vertex", "aws", "foundry"], + "default": "anthropic", + }, + "description": "Which client to build: anthropic (first-party API, default), bedrock (Amazon Bedrock), vertex (Google Vertex AI), aws (Claude Platform on AWS) or foundry (Microsoft Foundry).", + }, + "model": { + "label": "Default Model", + "schema": {"type": ["string", "null"]}, + "description": "Default model id used whenever an operator or hook call does not pass model (for example hook.create_message(...)). Falls back to claude-opus-4-8.", + }, + "aws_region": { + "label": "AWS Region", + "schema": {"type": ["string", "null"]}, + "description": "AWS region for the bedrock platform (for example us-east-1).", + }, + "project_id": { + "label": "GCP Project ID", + "schema": {"type": ["string", "null"]}, + "description": "Google Cloud project id for the vertex platform.", + }, + "region": { + "label": "GCP Region", + "schema": {"type": ["string", "null"]}, + "description": "Google Cloud region for the vertex platform (for example us-east5).", + }, + "resource": { + "label": "Azure Resource", + "schema": {"type": ["string", "null"]}, + "description": "Azure resource name for the foundry platform.", + }, + }, + "ui-field-behaviour": { + "hidden-fields": ["schema", "port", "login"], + "relabeling": {"password": "API Key", "host": "Base URL"}, + "placeholders": { + "extra": '{"anthropic_client_kwargs": {"timeout": 30, "max_retries": 5}, "workload_identity": {"federation_rule_id": "fdrl_...", "organization_id": "...", "service_account_id": "svac_...", "workspace_id": "wrkspc_...", "identity_token_file": "/var/run/secrets/anthropic.com/token"}}' + }, + }, + } + ], + } diff --git a/providers/anthropic/src/airflow/providers/anthropic/hooks/__init__.py b/providers/anthropic/src/airflow/providers/anthropic/hooks/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/anthropic/src/airflow/providers/anthropic/hooks/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/anthropic/src/airflow/providers/anthropic/hooks/anthropic.py b/providers/anthropic/src/airflow/providers/anthropic/hooks/anthropic.py new file mode 100644 index 0000000000000..6c576d667f86c --- /dev/null +++ b/providers/anthropic/src/airflow/providers/anthropic/hooks/anthropic.py @@ -0,0 +1,569 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import logging +import time +from enum import Enum +from functools import cached_property +from typing import TYPE_CHECKING, Any, cast + +from anthropic import ( + Anthropic, + AnthropicAWS, + AnthropicBedrock, + AnthropicFoundry, + AnthropicVertex, + IdentityTokenFile, + WorkloadIdentityCredentials, +) + +from airflow.providers.anthropic.exceptions import ( + AnthropicAgentSessionError, + AnthropicAgentSessionTimeout, + AnthropicBatchJobError, + AnthropicBatchTimeout, + AnthropicError, +) +from airflow.providers.common.compat.sdk import AirflowSkipException, BaseHook + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + + from anthropic.types import Message + from anthropic.types.beta import ( + BetaEnvironment, + BetaManagedAgentsAgent, + BetaManagedAgentsSession, + environment_create_params, + ) + from anthropic.types.beta.sessions import BetaManagedAgentsEventParams + from anthropic.types.messages import MessageBatch, MessageBatchIndividualResponse + from anthropic.types.messages.batch_create_params import Request + +#: Default model used when an operator or hook caller does not specify one. +#: Prefer configuring the model on the connection so it can be updated without +#: a provider release when this model ID is retired. +DEFAULT_MODEL = "claude-opus-4-8" + +#: Platforms that serve the first-party-only endpoints (Message Batches, token +#: counting, the Models API). Amazon Bedrock, Google Vertex AI and Microsoft +#: Foundry do not serve these, so the hook fails fast rather than surfacing a +#: raw ``404`` from the SDK. +FIRST_PARTY_PLATFORMS = frozenset({"anthropic", "aws"}) + +AnthropicClient = Anthropic | AnthropicBedrock | AnthropicVertex | AnthropicAWS | AnthropicFoundry + + +class BatchStatus(str, Enum): + """Top-level ``processing_status`` of an Anthropic Message Batch.""" + + IN_PROGRESS = "in_progress" + CANCELING = "canceling" + ENDED = "ended" + + @classmethod + def is_in_progress(cls, status: str) -> bool: + """Return ``True`` while the batch has not reached the terminal ``ended`` status.""" + return status != cls.ENDED + + +class SessionStatus(str, Enum): + """Status of a Managed Agents session.""" + + RESCHEDULING = "rescheduling" + RUNNING = "running" + IDLE = "idle" + TERMINATED = "terminated" + + @classmethod + def is_terminal(cls, status: str) -> bool: + """ + Return ``True`` once the session has stopped working. + + ``idle`` means the agent finished its turn (done, for an autonomous run); + ``terminated`` is an unrecoverable failure. Both stop the wait. + """ + return status in (cls.IDLE, cls.TERMINATED) + + +#: ``outcome_evaluations[].result`` values that mean the outcome did NOT succeed. +OUTCOME_FAILURE_RESULTS = frozenset({"failed", "max_iterations_reached", "interrupted"}) + + +def evaluate_session_state( + session: BetaManagedAgentsSession, *, expect_outcome: bool +) -> tuple[bool, str | None, bool]: + """ + Judge a polled session from its object fields alone. + + Returns ``(done, error_message, needs_event_check)``. ``done=False`` means keep + polling. ``needs_event_check=True`` means the session is ``idle`` on a ``message`` + run and the object can't say *why* — the caller must inspect the event log (see + :meth:`AnthropicHook.poll_session_completion`). + + The ``status`` field can't distinguish a genuine ``end_turn`` from ``requires_action`` + or ``retries_exhausted``, nor a just-created ``idle``. For an outcome run the true + verdict is in ``outcome_evaluations`` (judged here, which also defeats the start race). + """ + if session.status == SessionStatus.TERMINATED: + return True, f"Session {session.id} terminated.", False + if session.status != SessionStatus.IDLE: + return False, None, False + if not expect_outcome: + return False, None, True + for evaluation in session.outcome_evaluations: + if evaluation.result == "satisfied": + return True, None, False + if evaluation.result in OUTCOME_FAILURE_RESULTS: + return True, f"Outcome not satisfied for session {session.id}: {evaluation.result}.", False + # idle but no terminal outcome verdict yet (e.g. the run has not started) + return False, None, False + + +def evaluate_batch_counts( + *, + batch_id: str | None, + canceled: int, + errored: int, + expired: int, + succeeded: int, + fail_on_partial_error: bool, +) -> None: + """ + Apply the success/skip/fail policy for a terminal batch's request counts. + + Lives in the hook module so both :class:`AnthropicBatchOperator` and + :class:`~airflow.providers.anthropic.sensors.anthropic.AnthropicBatchSensor` share it + without an operator/sensor cross-import. Raises ``AirflowSkipException`` for a + fully-cancelled batch, ``AnthropicBatchJobError`` when ``fail_on_partial_error`` and any + request failed, otherwise returns (logging a warning for partial failures). + """ + total = canceled + errored + expired + succeeded + if total and canceled == total: + raise AirflowSkipException(f"Batch {batch_id} was fully cancelled.") + failed = errored + expired + if failed: + message = ( + f"Batch {batch_id} ended with {failed} failed request(s) " + f"(errored={errored}, expired={expired}, succeeded={succeeded})." + ) + if fail_on_partial_error: + raise AnthropicBatchJobError(message) + logger.warning("%s Successful results are still available.", message) + + +class AnthropicHook(BaseHook): + """ + Use the Anthropic SDK to interact with the Claude API. + + The connection's ``password`` is used as the API key and ``host`` as an optional + base URL (for gateways/proxies). The ``extra`` field selects the platform client + and passes platform-specific configuration: + + - ``platform``: one of ``anthropic`` (default), ``bedrock``, ``vertex``, ``aws``, ``foundry``. + - ``model``: default model id used when an operator/hook call omits ``model`` (lets you + change the model without editing Dags); falls back to :data:`DEFAULT_MODEL`. + - ``aws_region``: region for the ``bedrock`` platform. + - ``project_id`` / ``region``: project and region for the ``vertex`` platform. + - ``resource``: Azure resource name for the ``foundry`` platform. + - ``anthropic_client_kwargs``: extra keyword arguments forwarded to the client + constructor (e.g. ``timeout``, ``max_retries``, ``default_headers``). + - ``workload_identity``: configure `Workload Identity Federation + `__ + (keyless OIDC auth) with ``identity_token_file``, ``federation_rule_id``, + ``organization_id``, ``service_account_id`` and optional ``workspace_id`` / ``scope``. + + When the ``anthropic`` platform has no API Key and no ``workload_identity`` block, the + client is built with no static credential so the SDK resolves them from the environment + — supporting env-driven Workload Identity Federation and ``ant`` profiles. + + .. seealso:: https://docs.claude.com/en/api/client-sdks + + :param conn_id: :ref:`Anthropic connection id `. + """ + + conn_name_attr = "conn_id" + default_conn_name = "anthropic_default" + conn_type = "anthropic" + hook_name = "Anthropic" + + def __init__(self, conn_id: str = default_conn_name, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.conn_id = conn_id + + @cached_property + def _connection(self): + return self.get_connection(self.conn_id) + + @cached_property + def platform(self) -> str: + """Return the configured platform (defaults to ``anthropic``).""" + return (self._connection.extra_dejson.get("platform") or "anthropic").lower() + + @cached_property + def default_model(self) -> str: + """Default model id — connection ``extra['model']`` if set, else :data:`DEFAULT_MODEL`.""" + return self._connection.extra_dejson.get("model") or DEFAULT_MODEL + + @cached_property + def conn(self) -> AnthropicClient: + """Return the Anthropic client for the configured platform.""" + return self.get_conn() + + def get_conn(self) -> AnthropicClient: + """Build and return the Anthropic client for the configured platform.""" + conn = self._connection + extras = conn.extra_dejson + client_kwargs = dict(extras.get("anthropic_client_kwargs", {})) + platform = self.platform + self.log.debug("Building Anthropic client for platform %r (conn_id=%s)", platform, self.conn_id) + if platform == "bedrock": + return AnthropicBedrock(aws_region=extras.get("aws_region"), **client_kwargs) + if platform == "vertex": + return AnthropicVertex( + project_id=extras.get("project_id"), region=extras.get("region"), **client_kwargs + ) + if platform == "aws": + return AnthropicAWS(**client_kwargs) + if platform == "foundry": + api_key = client_kwargs.pop("api_key", None) or conn.password + return AnthropicFoundry(api_key=api_key, resource=extras.get("resource"), **client_kwargs) + if platform != "anthropic": + raise AnthropicError( + f"Unknown Anthropic platform {platform!r}. " + "Expected one of: anthropic, bedrock, vertex, aws, foundry." + ) + base_url = client_kwargs.pop("base_url", None) or conn.host or None + wif = extras.get("workload_identity") + if wif: + return Anthropic( + credentials=self._workload_identity_credentials(wif), base_url=base_url, **client_kwargs + ) + api_key = client_kwargs.pop("api_key", None) or conn.password + if api_key: + return Anthropic(api_key=api_key, base_url=base_url, **client_kwargs) + # No static key and no explicit federation config: let the SDK resolve credentials + # from the environment, which supports env-driven Workload Identity Federation + # (ANTHROPIC_FEDERATION_RULE_ID etc.) and ``ant`` profiles. + return Anthropic(base_url=base_url, **client_kwargs) + + @staticmethod + def _workload_identity_credentials(wif: dict[str, Any]) -> WorkloadIdentityCredentials: + """ + Build a WIF credential from the connection ``extra['workload_identity']`` mapping. + + Exchanges a short-lived OIDC token (read from ``identity_token_file``) for an + Anthropic access token. See + https://platform.claude.com/docs/en/manage-claude/workload-identity-federation. + """ + kwargs: dict[str, Any] = { + "identity_token_provider": IdentityTokenFile(wif["identity_token_file"]), + "federation_rule_id": wif["federation_rule_id"], + "organization_id": wif["organization_id"], + "service_account_id": wif["service_account_id"], + } + if wif.get("workspace_id"): + kwargs["workspace_id"] = wif["workspace_id"] + if wif.get("scope"): + kwargs["scope"] = wif["scope"] + return WorkloadIdentityCredentials(**kwargs) + + def _require_first_party(self, feature: str) -> None: + if self.platform not in FIRST_PARTY_PLATFORMS: + raise AnthropicError( + f"{feature} is not available on the {self.platform!r} platform. " + "Use the first-party Anthropic API (platform='anthropic') or " + "Claude Platform on AWS (platform='aws')." + ) + + @property + def _first_party_conn(self) -> Anthropic: + """ + Client cast to the first-party type for endpoints only it exposes. + + Callers must guard with :meth:`_require_first_party` first; the Bedrock/Vertex/ + Foundry clients don't expose ``beta.agents``/``beta.sessions``/``models``. + """ + return cast("Anthropic", self.conn) + + def test_connection(self) -> tuple[bool, str]: + """Test the Anthropic connection.""" + try: + if self.platform in FIRST_PARTY_PLATFORMS: + # Narrowed by the platform guard: only the first-party / AWS clients, + # which expose the Models API, reach this branch. + self._first_party_conn.models.list() + return True, "Connection established!" + # models.list() is not served on bedrock/vertex/foundry; building the + # client validates the configuration without a paid request. + self.get_conn() + return True, f"Connection configured for platform {self.platform!r} (no live check available)." + except Exception as e: + return False, str(e) + + def create_message( + self, + messages: list[dict[str, Any]], + model: str | None = None, + max_tokens: int = 1024, + system: str | None = None, + **kwargs: Any, + ) -> Message: + """ + Create a single message response (one-shot ``messages.create``). + + :param messages: The conversation so far, as a list of message dicts. + :param model: Model ID to use. Defaults to :attr:`default_model` (the connection's + ``extra['model']`` or :data:`DEFAULT_MODEL`). + :param max_tokens: Maximum number of tokens to generate. + :param system: Optional system prompt. + """ + params: dict[str, Any] = { + "model": model or self.default_model, + "max_tokens": max_tokens, + "messages": messages, + **kwargs, + } + if system is not None: + params["system"] = system + return self.conn.messages.create(**params) + + def count_tokens( + self, + messages: list[dict[str, Any]], + model: str | None = None, + system: str | None = None, + **kwargs: Any, + ) -> int: + """Return the number of input tokens the given request would consume.""" + self._require_first_party("Token counting") + params: dict[str, Any] = {"model": model or self.default_model, "messages": messages, **kwargs} + if system is not None: + params["system"] = system + return self.conn.messages.count_tokens(**params).input_tokens + + def create_batch(self, requests: list[dict[str, Any]]) -> MessageBatch: + """ + Submit a Message Batch. + + :param requests: A list of ``{"custom_id": str, "params": {...}}`` dicts, where + ``params`` is a ``messages.create`` payload (``model``, ``max_tokens``, + ``messages``, ...). + """ + self._require_first_party("The Message Batches API") + # ``Request`` is a TypedDict, so the plain dicts callers build match structurally. + return self.conn.messages.batches.create(requests=cast("Iterable[Request]", requests)) + + def get_batch(self, batch_id: str) -> MessageBatch: + """Retrieve a Message Batch by ID.""" + self._require_first_party("The Message Batches API") + return self.conn.messages.batches.retrieve(batch_id) + + def cancel_batch(self, batch_id: str) -> MessageBatch: + """Request cancellation of a Message Batch.""" + self._require_first_party("The Message Batches API") + return self.conn.messages.batches.cancel(batch_id) + + def list_batches(self, **kwargs: Any) -> Any: + """Return a (paginated) list of Message Batches.""" + self._require_first_party("The Message Batches API") + return self.conn.messages.batches.list(**kwargs) + + def stream_batch_results(self, batch_id: str) -> Iterator[MessageBatchIndividualResponse]: + """ + Return a streaming iterator of per-request results, keyed by ``custom_id``. + + Results stream from the API and arrive in **arbitrary order** — key them by + ``result.custom_id``, never by position. Results are available for 29 days + after the batch is created. The result set can be very large: iterate and + persist to object storage; do not materialize it into XCom. + """ + # Return (don't ``yield``) so the platform guard fails fast at call time + # rather than only when the caller starts iterating. + self._require_first_party("The Message Batches API") + return self.conn.messages.batches.results(batch_id) + + def wait_for_batch( + self, batch_id: str, wait_seconds: float = 3, timeout: float = 24 * 60 * 60 + ) -> MessageBatch: + """ + Poll a batch synchronously until it reaches the terminal ``ended`` status. + + :param batch_id: The batch to wait for. + :param wait_seconds: Seconds to sleep between polls. + :param timeout: Maximum seconds to wait before raising :class:`AnthropicBatchTimeout`. + :return: The terminal :class:`~anthropic.types.messages.MessageBatch`. + """ + start = time.monotonic() + while True: + batch = self.get_batch(batch_id) + self.log.debug("Batch %s status=%s", batch_id, batch.processing_status) + if not BatchStatus.is_in_progress(batch.processing_status): + return batch + if time.monotonic() - start > timeout: + raise AnthropicBatchTimeout( + f"Batch {batch_id} did not reach a terminal status within {timeout} seconds." + ) + time.sleep(wait_seconds) + + # --- Managed Agents ------------------------------------------------------- + # Agents and environments are persisted, reusable resources: create them once + # (these helpers, the ``ant`` CLI, or a setup script) and store the IDs. The + # operator references those IDs; it never creates an agent per run. + + def create_agent(self, name: str, model: str | None = None, **kwargs: Any) -> BetaManagedAgentsAgent: + """ + Create a (reusable, versioned) Managed Agents agent. One-time setup. + + ``model`` defaults to :attr:`default_model` (the connection's ``extra['model']`` + or :data:`DEFAULT_MODEL`). + """ + self._require_first_party("Managed Agents") + agent = self._first_party_conn.beta.agents.create( + name=name, model=model or self.default_model, **kwargs + ) + self.log.debug("Created agent %s (name=%r, model=%s)", agent.id, name, model or self.default_model) + return agent + + def create_environment( + self, name: str, config: dict[str, Any] | None = None, **kwargs: Any + ) -> BetaEnvironment: + """Create a (reusable) environment for agent sessions. One-time setup.""" + self._require_first_party("Managed Agents") + if config is None: + config = {"type": "cloud", "networking": {"type": "unrestricted"}} + environment = self._first_party_conn.beta.environments.create( + name=name, config=cast("environment_create_params.Config", config), **kwargs + ) + self.log.debug("Created environment %s (name=%r)", environment.id, name) + return environment + + def create_session(self, agent: str, environment_id: str, **kwargs: Any) -> BetaManagedAgentsSession: + """Start a session against a pre-created agent + environment.""" + self._require_first_party("Managed Agents") + return self._first_party_conn.beta.sessions.create( + agent=agent, environment_id=environment_id, **kwargs + ) + + def get_session(self, session_id: str) -> BetaManagedAgentsSession: + """Retrieve a session (carries its current ``status``).""" + self._require_first_party("Managed Agents") + return self._first_party_conn.beta.sessions.retrieve(session_id) + + def send_event(self, session_id: str, event: dict[str, Any]) -> Any: + """Send a single event (e.g. a ``user.message`` or ``user.define_outcome``).""" + self._require_first_party("Managed Agents") + # Event dicts callers build match the SDK's TypedDict union structurally. + return self._first_party_conn.beta.sessions.events.send( + session_id, events=cast("list[BetaManagedAgentsEventParams]", [event]) + ) + + def archive_session(self, session_id: str) -> Any: + """Archive a session (frees the server-side container). Best-effort teardown.""" + self._require_first_party("Managed Agents") + return self._first_party_conn.beta.sessions.archive(session_id) + + def _latest_idle_reason(self, session_id: str, kickoff_event_id: str | None) -> str | None: + """ + Return the ``stop_reason`` of the newest ``session.status_idle`` event, or ``None``. + + Walks the event log newest-first. Returns ``None`` if the kickoff event is the most + recent event (the agent has not responded yet — defeats the start race) or no idle + event is found in the scan window. + """ + # The SDK cursor auto-paginates (page size 20); cap the walk at 100 events so a + # long event log can't make one poll iterate unboundedly. + examined = 0 + for event in self._first_party_conn.beta.sessions.events.list(session_id, order="desc", limit=20): + if kickoff_event_id is not None and event.id == kickoff_event_id: + return None + if event.type == "session.status_idle": + return event.stop_reason.type + examined += 1 + if examined >= 100: + break + return None + + def poll_session_completion( + self, session_id: str, *, expect_outcome: bool = False, kickoff_event_id: str | None = None + ) -> tuple[bool, str | None]: + """ + Return ``(done, error_message)`` for one poll of a session. + + Combines the session object (status / outcome verdict) with the event log + (``stop_reason`` of the latest idle) so a ``message`` run distinguishes genuine + ``end_turn`` completion from ``requires_action`` / ``retries_exhausted``. + """ + session = self.get_session(session_id) + done, error_message, needs_event_check = evaluate_session_state( + session, expect_outcome=expect_outcome + ) + self.log.debug( + "Session %s status=%s done=%s needs_event_check=%s", + session_id, + session.status, + done, + needs_event_check, + ) + if not needs_event_check: + return done, error_message + reason = self._latest_idle_reason(session_id, kickoff_event_id) + if reason is None: + return False, None + if reason == "end_turn": + return True, None + return True, ( + f"Session {session_id} is idle but did not complete ({reason}); " + "configure an autonomous agent or use an outcome run." + ) + + def wait_for_session( + self, + session_id: str, + expect_outcome: bool = False, + kickoff_event_id: str | None = None, + poll_interval: float = 30, + timeout: float = 24 * 60 * 60, + ) -> None: + """ + Poll a session synchronously until it completes. + + :param session_id: The session to wait for. + :param expect_outcome: Whether the session is running a ``user.define_outcome`` loop + (completion judged from ``outcome_evaluations``). + :param kickoff_event_id: ID of the kickoff event, used to correlate the terminal + idle event on a ``message`` run (defeats the start race). + :param poll_interval: Seconds to sleep between polls. + :param timeout: Maximum seconds to wait before raising :class:`AnthropicAgentSessionTimeout`. + """ + start = time.monotonic() + while True: + done, error_message = self.poll_session_completion( + session_id, expect_outcome=expect_outcome, kickoff_event_id=kickoff_event_id + ) + if done: + if error_message: + raise AnthropicAgentSessionError(error_message) + return + if time.monotonic() - start > timeout: + raise AnthropicAgentSessionTimeout( + f"Session {session_id} did not reach a terminal status within {timeout} seconds." + ) + time.sleep(poll_interval) diff --git a/providers/anthropic/src/airflow/providers/anthropic/operators/__init__.py b/providers/anthropic/src/airflow/providers/anthropic/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/anthropic/src/airflow/providers/anthropic/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/anthropic/src/airflow/providers/anthropic/operators/agent.py b/providers/anthropic/src/airflow/providers/anthropic/operators/agent.py new file mode 100644 index 0000000000000..4bf04b5218709 --- /dev/null +++ b/providers/anthropic/src/airflow/providers/anthropic/operators/agent.py @@ -0,0 +1,219 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import time +from collections.abc import Sequence +from datetime import timedelta +from functools import cached_property +from typing import TYPE_CHECKING, Any + +from airflow.providers.anthropic.exceptions import AnthropicAgentSessionError, AnthropicAgentSessionTimeout +from airflow.providers.anthropic.hooks.anthropic import AnthropicHook +from airflow.providers.anthropic.triggers.agent import AnthropicAgentSessionTrigger +from airflow.providers.common.compat.sdk import BaseOperator, conf + +if TYPE_CHECKING: + from airflow.providers.common.compat.sdk import Context + + +class AnthropicAgentSessionOperator(BaseOperator): + """ + Run a Managed Agents session against a pre-created agent and environment. + + Anthropic runs the agent loop server-side; the worker creates a session, sends the + initial instruction, and waits for the session to reach a terminal status. In + deferrable mode it releases the worker slot while a trigger polls the session status. + + Provide exactly one of ``message`` (a single user turn) or ``outcome`` (a + ``user.define_outcome`` rubric that the agent iterates against until satisfied). + + .. important:: + This operator is for **autonomous** agents — configure the agent without + client-side custom tools or an ``always_ask`` permission policy. + + Completion is detected accurately for both modes. A ``message`` run reads the + terminal ``session.status_idle`` event's ``stop_reason`` (correlated against the + kickoff event to avoid a start-race false positive): ``end_turn`` succeeds, while + ``requires_action`` (the agent is blocked on input) and ``retries_exhausted`` + raise an error rather than silently passing. An ``outcome`` run is judged from the + session's ``outcome_evaluations`` verdict (``satisfied`` vs. + ``failed``/``max_iterations_reached``/``interrupted``). + + Agents and environments are created once (see + :meth:`~airflow.providers.anthropic.hooks.anthropic.AnthropicHook.create_agent`), + not per task run. + + Outputs the agent writes to ``/mnt/session/outputs/`` are retrieved afterwards via the + Files API (``scope_id=``); the operator returns the **session ID only**. + + .. seealso:: + For more information, take a look at the guide: + :ref:`howto/operator:AnthropicAgentSessionOperator` + + :param agent_id: ID of a pre-created agent. + :param environment_id: ID of a pre-created environment. + :param message: A single user message to start the session. Mutually exclusive with ``outcome``. + :param outcome: A ``user.define_outcome`` payload (``description``, ``rubric``, + optional ``max_iterations``). Mutually exclusive with ``message``. + :param conn_id: The Anthropic connection ID to use. + :param deferrable: Run the operator in deferrable mode. + :param poll_interval: Seconds between session status checks (both paths). + :param timeout: Seconds to wait for a terminal status. Defaults to 24 hours. In + deferrable mode the trigger enforces this and tears the session down on timeout; + set ``execution_timeout`` only for a shorter hard cap (which preempts that + graceful teardown). + :param vault_ids: Vault IDs providing MCP/credential access to the session. + :param session_resources: Session resources (files, GitHub repos, memory stores). Named + ``session_resources`` to avoid colliding with the reserved ``BaseOperator.resources``; + forwarded to ``sessions.create`` as ``resources``. + :param session_kwargs: Extra keyword arguments forwarded to ``sessions.create``. + """ + + template_fields: Sequence[str] = ("agent_id", "environment_id", "message", "outcome") + + def __init__( + self, + *, + agent_id: str, + environment_id: str, + message: str | None = None, + outcome: dict[str, Any] | None = None, + conn_id: str = AnthropicHook.default_conn_name, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), + poll_interval: float = 30, + timeout: float = 24 * 60 * 60, + vault_ids: list[str] | None = None, + session_resources: list[dict[str, Any]] | None = None, + session_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + if (message is None) == (outcome is None): + raise ValueError("Provide exactly one of 'message' or 'outcome'.") + if outcome is not None and "rubric" not in outcome: + raise ValueError("'outcome' must include a 'rubric' (with 'description').") + self.agent_id = agent_id + self.environment_id = environment_id + self.message = message + self.outcome = outcome + self.conn_id = conn_id + self.deferrable = deferrable + self.poll_interval = poll_interval + self.timeout = timeout + self.vault_ids = vault_ids + self.session_resources = session_resources + self.session_kwargs = session_kwargs or {} + self.session_id: str | None = None + + @cached_property + def hook(self) -> AnthropicHook: + """Return an instance of the AnthropicHook.""" + return AnthropicHook(conn_id=self.conn_id) + + def execute(self, context: Context) -> str | None: + create_kwargs: dict[str, Any] = dict(self.session_kwargs) + if self.vault_ids: + create_kwargs["vault_ids"] = self.vault_ids + if self.session_resources: + create_kwargs["resources"] = self.session_resources + session = self.hook.create_session( + agent=self.agent_id, environment_id=self.environment_id, **create_kwargs + ) + self.session_id = session.id + context["ti"].xcom_push(key="session_id", value=session.id) + self.log.info("Started Anthropic session %s for agent %s", session.id, self.agent_id) + + if self.outcome is not None: + response = self.hook.send_event(session.id, {"type": "user.define_outcome", **self.outcome}) + else: + response = self.hook.send_event( + session.id, + {"type": "user.message", "content": [{"type": "text", "text": self.message}]}, + ) + # Correlate completion against the kickoff event so a message run is not fooled by a + # just-created idle session (the start race). + sent = response.data + kickoff_event_id = sent[-1].id if sent else None + + expect_outcome = self.outcome is not None + if self.deferrable: + # Backstop the deferral slightly beyond the trigger's own end_time so the + # trigger's clean "timeout" event (which tears the session down) wins the race + # rather than a generic AirflowTaskTimeout. A user-set execution_timeout still + # applies as a shorter hard cap (and then teardown is skipped — documented). + self.defer( + timeout=self.execution_timeout or timedelta(seconds=self.timeout + self.poll_interval + 60), + trigger=AnthropicAgentSessionTrigger( + conn_id=self.conn_id, + session_id=session.id, + poll_interval=self.poll_interval, + end_time=time.time() + self.timeout, + expect_outcome=expect_outcome, + kickoff_event_id=kickoff_event_id, + ), + method_name="execute_complete", + ) + + self.log.info("Waiting for session %s to complete", session.id) + try: + self.hook.wait_for_session( + session.id, + expect_outcome=expect_outcome, + kickoff_event_id=kickoff_event_id, + poll_interval=self.poll_interval, + timeout=self.timeout, + ) + except AnthropicAgentSessionTimeout: + # Mirror the deferrable execute_complete: tear the session down on a sync + # timeout too, so its server-side container does not linger. + self._archive_session(session.id) + raise + return session.id + + def execute_complete(self, context: Context, event: Any = None) -> str: + if not event: + raise AnthropicAgentSessionError("Trigger resumed without an event payload.") + # The deferred task is a fresh instance; restore the session id from the event. + self.session_id = event["session_id"] + status = event["status"] + if status == "timeout": + self._archive_session(self.session_id) + raise AnthropicAgentSessionTimeout(event["message"]) + if status == "error": + raise AnthropicAgentSessionError(event["message"]) + self.log.info("Session %s completed.", self.session_id) + return self.session_id + + def _archive_session(self, session_id: str | None) -> None: + """Best-effort teardown of the server-side session (frees its container).""" + if not session_id: + return + try: + self.hook.archive_session(session_id) + except Exception as e: + self.log.warning("Failed to archive session %s: %s", session_id, e) + + def on_kill(self) -> None: + """ + Archive the session if the (non-deferred) task is killed. + + Only fires while the worker process is alive — i.e. the synchronous path + (``deferrable=False``). A deferred task has released its slot, so killing it does + not run ``on_kill``; archive such a session manually via the hook. + """ + self._archive_session(self.session_id) diff --git a/providers/anthropic/src/airflow/providers/anthropic/operators/anthropic.py b/providers/anthropic/src/airflow/providers/anthropic/operators/anthropic.py new file mode 100644 index 0000000000000..fd5429ea76dbb --- /dev/null +++ b/providers/anthropic/src/airflow/providers/anthropic/operators/anthropic.py @@ -0,0 +1,200 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import time +from collections.abc import Sequence +from datetime import timedelta +from functools import cached_property +from typing import TYPE_CHECKING, Any + +from airflow.providers.anthropic.exceptions import AnthropicBatchJobError, AnthropicBatchTimeout +from airflow.providers.anthropic.hooks.anthropic import AnthropicHook, evaluate_batch_counts +from airflow.providers.anthropic.triggers.anthropic import AnthropicBatchTrigger +from airflow.providers.common.compat.sdk import BaseOperator, conf + +if TYPE_CHECKING: + from airflow.providers.common.compat.sdk import Context + + +class AnthropicBatchOperator(BaseOperator): + """ + Submit an Anthropic Message Batch and wait for it to complete. + + Message Batches process many ``messages.create`` requests asynchronously at 50% of + standard cost; most complete within an hour (24h SLA). This operator submits the + batch and, in deferrable mode, releases the worker slot while a trigger polls for + completion. + + The operator returns the **batch ID only** — never the results. Pull results with + :meth:`~airflow.providers.anthropic.hooks.anthropic.AnthropicHook.stream_batch_results` + and persist them to object storage; results can be very large and must not be pushed + to XCom. Results are retained for 29 days after the batch is created. + + .. note:: + A retry re-submits a brand-new batch. Prefer ``retries=0`` on this task (the + submitted ``batch_id`` is pushed to XCom under key ``batch_id`` immediately, so + a crashed run never loses track of an in-flight batch). + + .. seealso:: + For more information, take a look at the guide: + :ref:`howto/operator:AnthropicBatchOperator` + + :param requests: A list of ``{"custom_id": str, "params": {...}}`` dicts, where + ``params`` is a ``messages.create`` payload (``model``, ``max_tokens``, ``messages``, ...). + :param conn_id: The Anthropic connection ID to use. + :param deferrable: Run the operator in deferrable mode. + :param poll_interval: Seconds between status checks, in both the synchronous and + deferrable paths. + :param timeout: Seconds to wait for the batch to reach a terminal status. Defaults to + 24 hours (the Message Batches SLA). In deferrable mode this also bounds the + deferral; set ``execution_timeout`` only if you want a shorter hard cap (note a + shorter ``execution_timeout`` preempts the graceful cancel-on-timeout path). + :param wait_for_completion: Whether to wait for the batch to complete. If ``False``, + the operator returns the batch ID immediately after submission. + :param fail_on_partial_error: If ``True``, fail the task when any request errored or + expired. Defaults to ``False`` (succeed and log a warning so the successful + results are not discarded). + """ + + template_fields: Sequence[str] = ("requests",) + + def __init__( + self, + requests: list[dict[str, Any]], + conn_id: str = AnthropicHook.default_conn_name, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), + poll_interval: float = 60, + timeout: float = 24 * 60 * 60, + wait_for_completion: bool = True, + fail_on_partial_error: bool = False, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self.requests = requests + self.conn_id = conn_id + self.deferrable = deferrable + self.poll_interval = poll_interval + self.timeout = timeout + self.wait_for_completion = wait_for_completion + self.fail_on_partial_error = fail_on_partial_error + self.batch_id: str | None = None + + @cached_property + def hook(self) -> AnthropicHook: + """Return an instance of the AnthropicHook.""" + return AnthropicHook(conn_id=self.conn_id) + + def execute(self, context: Context) -> str | None: + if not self.requests: + raise ValueError("AnthropicBatchOperator requires at least one request; got an empty list.") + batch = self.hook.create_batch(self.requests) + self.batch_id = batch.id + # Push immediately so a crash between submit and completion never loses the batch. + context["ti"].xcom_push(key="batch_id", value=batch.id) + self.log.info("Submitted Anthropic Message Batch %s (%d requests)", batch.id, len(self.requests)) + + if not self.wait_for_completion: + return self.batch_id + + if self.deferrable: + self.defer( + # Backstop the deferral slightly beyond the trigger's own end_time so the + # trigger's clean "timeout" event (which cancels the batch) wins over a + # generic AirflowTaskTimeout. A user-set execution_timeout still applies + # as a shorter hard cap. + timeout=self.execution_timeout or timedelta(seconds=self.timeout + self.poll_interval + 60), + trigger=AnthropicBatchTrigger( + conn_id=self.conn_id, + batch_id=self.batch_id, + poll_interval=self.poll_interval, + end_time=time.time() + self.timeout, + ), + method_name="execute_complete", + ) + + self.log.info("Waiting for batch %s to complete", self.batch_id) + try: + batch = self.hook.wait_for_batch( + self.batch_id, wait_seconds=self.poll_interval, timeout=self.timeout + ) + except AnthropicBatchTimeout: + # Mirror the deferrable execute_complete: tear down the still-running batch + # before the task fails, so a sync timeout does not leave it billing. + self.log.warning("Batch %s timed out; requesting cancellation.", self.batch_id) + self._cancel_batch_quietly() + raise + counts = batch.request_counts + self._apply_policy(counts.canceled, counts.errored, counts.expired, counts.succeeded) + return self.batch_id + + def execute_complete(self, context: Context, event: Any = None) -> str: + """ + Resume after the trigger fires. + + The deferred task is a fresh instance, so the batch ID is read from the event, + not ``self.batch_id``. + """ + self.batch_id = event["batch_id"] + status = event["status"] + if status == "timeout": + self.log.warning("Batch %s timed out; requesting cancellation.", self.batch_id) + self._cancel_batch_quietly() + raise AnthropicBatchTimeout(event["message"]) + if status == "error": + raise AnthropicBatchJobError(event["message"]) + + counts = event.get("request_counts") or {} + self._apply_policy( + counts.get("canceled", 0), + counts.get("errored", 0), + counts.get("expired", 0), + counts.get("succeeded", 0), + ) + self.log.info("%s completed successfully.", self.task_id) + return self.batch_id + + def _apply_policy(self, canceled: int, errored: int, expired: int, succeeded: int) -> None: + evaluate_batch_counts( + batch_id=self.batch_id, + canceled=canceled, + errored=errored, + expired=expired, + succeeded=succeeded, + fail_on_partial_error=self.fail_on_partial_error, + ) + + def on_kill(self) -> None: + """ + Cancel the batch if the (non-deferred) task is killed. + + This only fires while the worker process is alive — i.e. the synchronous path + (``deferrable=False``). A deferred task has released its worker slot, so killing + it does not run ``on_kill``; cancel such a batch manually via the hook. + """ + if self.batch_id: + self.log.info("on_kill: cancelling Anthropic batch %s", self.batch_id) + self._cancel_batch_quietly() + + def _cancel_batch_quietly(self) -> None: + """Best-effort batch cancellation for the timeout and kill paths.""" + if not self.batch_id: + return + try: + self.hook.cancel_batch(self.batch_id) + except Exception as e: + self.log.warning("Failed to cancel batch %s: %s", self.batch_id, e) diff --git a/providers/anthropic/src/airflow/providers/anthropic/sensors/__init__.py b/providers/anthropic/src/airflow/providers/anthropic/sensors/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/anthropic/src/airflow/providers/anthropic/sensors/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/anthropic/src/airflow/providers/anthropic/sensors/anthropic.py b/providers/anthropic/src/airflow/providers/anthropic/sensors/anthropic.py new file mode 100644 index 0000000000000..dbc0acb594eeb --- /dev/null +++ b/providers/anthropic/src/airflow/providers/anthropic/sensors/anthropic.py @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import time +from collections.abc import Sequence +from datetime import timedelta +from functools import cached_property +from typing import TYPE_CHECKING, Any + +from airflow.providers.anthropic.exceptions import AnthropicBatchJobError, AnthropicBatchTimeout +from airflow.providers.anthropic.hooks.anthropic import AnthropicHook, BatchStatus, evaluate_batch_counts +from airflow.providers.anthropic.triggers.anthropic import AnthropicBatchTrigger +from airflow.providers.common.compat.sdk import BaseSensorOperator, conf + +if TYPE_CHECKING: + from airflow.providers.common.compat.sdk import Context + + +class AnthropicBatchSensor(BaseSensorOperator): + """ + Wait for an already-submitted Anthropic Message Batch to reach a terminal status. + + Pairs with ``AnthropicBatchOperator(wait_for_completion=False)`` (or any out-of-band + submission) for a fire-and-forget submit + re-entrant await. Because the sensor only + polls an existing ``batch_id``, it is naturally idempotent across retries — unlike a + submit step, retrying it never creates a new batch. + + On a terminal batch it applies the same policy as the operator: a fully-cancelled + batch skips the task, and ``fail_on_partial_error`` controls whether errored/expired + requests fail it. + + .. seealso:: + For more information, take a look at the guide: + :ref:`howto/sensor:AnthropicBatchSensor` + + :param batch_id: The ID of the batch to wait for. + :param conn_id: The Anthropic connection ID to use. + :param deferrable: Run the sensor in deferrable mode (polls via a trigger). + :param fail_on_partial_error: If ``True``, fail when any request errored or expired. + Defaults to ``False`` (succeed and log a warning). + """ + + template_fields: Sequence[str] = ("batch_id",) + + def __init__( + self, + *, + batch_id: str, + conn_id: str = AnthropicHook.default_conn_name, + deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), + fail_on_partial_error: bool = False, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self.batch_id = batch_id + self.conn_id = conn_id + self.deferrable = deferrable + self.fail_on_partial_error = fail_on_partial_error + + @cached_property + def hook(self) -> AnthropicHook: + """Return an instance of the AnthropicHook.""" + return AnthropicHook(conn_id=self.conn_id) + + def poke(self, context: Context) -> bool: + batch = self.hook.get_batch(self.batch_id) + if BatchStatus.is_in_progress(batch.processing_status): + return False + counts = batch.request_counts + evaluate_batch_counts( + batch_id=self.batch_id, + canceled=counts.canceled, + errored=counts.errored, + expired=counts.expired, + succeeded=counts.succeeded, + fail_on_partial_error=self.fail_on_partial_error, + ) + return True + + def execute(self, context: Context) -> None: + if self.deferrable: + self.defer( + timeout=timedelta(seconds=self.timeout), + trigger=AnthropicBatchTrigger( + conn_id=self.conn_id, + batch_id=self.batch_id, + poll_interval=self.poke_interval, + end_time=time.time() + self.timeout, + ), + method_name="execute_complete", + ) + super().execute(context) + + def execute_complete(self, context: Context, event: Any = None) -> None: + status = event["status"] + if status == "timeout": + raise AnthropicBatchTimeout(event["message"]) + if status == "error": + raise AnthropicBatchJobError(event["message"]) + counts = event.get("request_counts") or {} + evaluate_batch_counts( + batch_id=event["batch_id"], + canceled=counts.get("canceled", 0), + errored=counts.get("errored", 0), + expired=counts.get("expired", 0), + succeeded=counts.get("succeeded", 0), + fail_on_partial_error=self.fail_on_partial_error, + ) + self.log.info("Batch %s reached a terminal status.", event["batch_id"]) diff --git a/providers/anthropic/src/airflow/providers/anthropic/triggers/__init__.py b/providers/anthropic/src/airflow/providers/anthropic/triggers/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/anthropic/src/airflow/providers/anthropic/triggers/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/anthropic/src/airflow/providers/anthropic/triggers/agent.py b/providers/anthropic/src/airflow/providers/anthropic/triggers/agent.py new file mode 100644 index 0000000000000..ae69cd826babe --- /dev/null +++ b/providers/anthropic/src/airflow/providers/anthropic/triggers/agent.py @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import asyncio +import time +from collections.abc import AsyncIterator +from typing import Any + +from airflow.providers.anthropic.hooks.anthropic import AnthropicHook +from airflow.triggers.base import BaseTrigger, TriggerEvent + +#: Consecutive failed polls tolerated before the trigger gives up (transient errors). +MAX_CONSECUTIVE_POLL_FAILURES = 5 + + +class AnthropicAgentSessionTrigger(BaseTrigger): + """ + Poll a Managed Agents session until it reaches a terminal status. + + :param conn_id: The Anthropic connection ID. + :param session_id: The session to poll. + :param poll_interval: Seconds to sleep between polls. + :param end_time: Wall-clock deadline (``time.time()`` epoch seconds). Wall-clock is + used deliberately: the trigger is serialized and may resume in a different + triggerer process, where a per-process ``time.monotonic()`` value is meaningless. + :param expect_outcome: Whether the session is running a ``user.define_outcome`` loop + (completion is then judged from ``outcome_evaluations``, not from ``idle`` alone). + :param kickoff_event_id: ID of the kickoff event, used to correlate the terminal idle + event on a ``message`` run (defeats the start race). + """ + + def __init__( + self, + conn_id: str, + session_id: str, + poll_interval: float, + end_time: float, + expect_outcome: bool = False, + kickoff_event_id: str | None = None, + ) -> None: + super().__init__() + self.conn_id = conn_id + self.session_id = session_id + self.poll_interval = poll_interval + self.end_time = end_time + self.expect_outcome = expect_outcome + self.kickoff_event_id = kickoff_event_id + + def serialize(self) -> tuple[str, dict[str, Any]]: + """Serialize AnthropicAgentSessionTrigger arguments and class path.""" + return ( + "airflow.providers.anthropic.triggers.agent.AnthropicAgentSessionTrigger", + { + "conn_id": self.conn_id, + "session_id": self.session_id, + "poll_interval": self.poll_interval, + "end_time": self.end_time, + "expect_outcome": self.expect_outcome, + "kickoff_event_id": self.kickoff_event_id, + }, + ) + + async def run(self) -> AsyncIterator[TriggerEvent]: + """Poll the session and yield exactly one terminal event.""" + hook = AnthropicHook(conn_id=self.conn_id) + consecutive_failures = 0 + while True: + try: + # poll_session_completion does blocking SDK HTTP calls; run off the event loop. + done, error_message = await asyncio.to_thread( + hook.poll_session_completion, + self.session_id, + expect_outcome=self.expect_outcome, + kickoff_event_id=self.kickoff_event_id, + ) + except Exception as e: + # Tolerate transient polling errors rather than failing the whole wait. + consecutive_failures += 1 + if consecutive_failures >= MAX_CONSECUTIVE_POLL_FAILURES or time.time() > self.end_time: + yield TriggerEvent({"status": "error", "session_id": self.session_id, "message": str(e)}) + return + self.log.warning("Polling session %s failed (%s); retrying.", self.session_id, e) + await asyncio.sleep(self.poll_interval) + continue + + consecutive_failures = 0 + if done: + if error_message: + yield TriggerEvent( + {"status": "error", "session_id": self.session_id, "message": error_message} + ) + else: + yield TriggerEvent( + { + "status": "success", + "session_id": self.session_id, + "message": f"Session {self.session_id} completed.", + } + ) + return + if time.time() > self.end_time: + yield TriggerEvent( + { + "status": "timeout", + "session_id": self.session_id, + "message": ( + f"Session {self.session_id} did not reach a terminal status " + "before the configured timeout." + ), + } + ) + return + await asyncio.sleep(self.poll_interval) diff --git a/providers/anthropic/src/airflow/providers/anthropic/triggers/anthropic.py b/providers/anthropic/src/airflow/providers/anthropic/triggers/anthropic.py new file mode 100644 index 0000000000000..83e99324e3ebf --- /dev/null +++ b/providers/anthropic/src/airflow/providers/anthropic/triggers/anthropic.py @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import asyncio +import time +from collections.abc import AsyncIterator +from typing import Any + +from airflow.providers.anthropic.hooks.anthropic import AnthropicHook, BatchStatus +from airflow.triggers.base import BaseTrigger, TriggerEvent + +#: Consecutive failed polls tolerated before the trigger gives up (transient errors). +MAX_CONSECUTIVE_POLL_FAILURES = 5 + + +class AnthropicBatchTrigger(BaseTrigger): + """ + Poll an Anthropic Message Batch until it reaches the terminal ``ended`` status. + + :param conn_id: The Anthropic connection ID. + :param batch_id: The batch to poll. + :param poll_interval: Seconds to sleep between polls. + :param end_time: Wall-clock deadline (``time.time()`` epoch seconds) after which a + ``timeout`` event is emitted. Wall-clock is used deliberately: the trigger is + serialized to the metadata DB and may resume in a different triggerer process, + so a per-process ``time.monotonic()`` value would not survive serialization. + """ + + def __init__(self, conn_id: str, batch_id: str, poll_interval: float, end_time: float) -> None: + super().__init__() + self.conn_id = conn_id + self.batch_id = batch_id + self.poll_interval = poll_interval + self.end_time = end_time + + def serialize(self) -> tuple[str, dict[str, Any]]: + """Serialize AnthropicBatchTrigger arguments and class path.""" + return ( + "airflow.providers.anthropic.triggers.anthropic.AnthropicBatchTrigger", + { + "conn_id": self.conn_id, + "batch_id": self.batch_id, + "poll_interval": self.poll_interval, + "end_time": self.end_time, + }, + ) + + async def run(self) -> AsyncIterator[TriggerEvent]: + """Poll the batch status and yield exactly one terminal event.""" + hook = AnthropicHook(conn_id=self.conn_id) + consecutive_failures = 0 + while True: + try: + # get_batch is a blocking SDK HTTP call; run it off the event loop so a + # single poll does not stall every other trigger on this triggerer. + batch = await asyncio.to_thread(hook.get_batch, self.batch_id) + except Exception as e: + # Tolerate transient polling errors rather than failing a (up to 24h) wait. + consecutive_failures += 1 + if consecutive_failures >= MAX_CONSECUTIVE_POLL_FAILURES or time.time() > self.end_time: + yield TriggerEvent({"status": "error", "batch_id": self.batch_id, "message": str(e)}) + return + self.log.warning("Polling batch %s failed (%s); retrying.", self.batch_id, e) + await asyncio.sleep(self.poll_interval) + continue + + consecutive_failures = 0 + self.log.debug("Batch %s status=%s", self.batch_id, batch.processing_status) + if not BatchStatus.is_in_progress(batch.processing_status): + counts = batch.request_counts + yield TriggerEvent( + { + "status": "success", + "batch_id": self.batch_id, + "message": f"Batch {self.batch_id} has ended.", + "request_counts": { + "succeeded": counts.succeeded, + "errored": counts.errored, + "canceled": counts.canceled, + "expired": counts.expired, + "processing": counts.processing, + }, + } + ) + return + if time.time() > self.end_time: + yield TriggerEvent( + { + "status": "timeout", + "batch_id": self.batch_id, + "message": ( + f"Batch {self.batch_id} did not reach a terminal status " + "before the configured timeout." + ), + } + ) + return + await asyncio.sleep(self.poll_interval) diff --git a/providers/anthropic/tests/conftest.py b/providers/anthropic/tests/conftest.py new file mode 100644 index 0000000000000..f56ccce0a3f69 --- /dev/null +++ b/providers/anthropic/tests/conftest.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +pytest_plugins = "tests_common.pytest_plugin" diff --git a/providers/anthropic/tests/system/__init__.py b/providers/anthropic/tests/system/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/anthropic/tests/system/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/anthropic/tests/system/anthropic/__init__.py b/providers/anthropic/tests/system/anthropic/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/anthropic/tests/system/anthropic/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/anthropic/tests/system/anthropic/example_anthropic_agent.py b/providers/anthropic/tests/system/anthropic/example_anthropic_agent.py new file mode 100644 index 0000000000000..88b8b2cfee79e --- /dev/null +++ b/providers/anthropic/tests/system/anthropic/example_anthropic_agent.py @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.sdk import dag, task + +ANTHROPIC_CONN_ID = "anthropic_default" + + +@dag(schedule=None, catchup=False) +def anthropic_managed_agent(): + @task + def setup_agent_and_environment() -> dict[str, str]: + # One-time setup helper — in production run this once and store the IDs, do not + # create a fresh agent every Dag run. Here we create them and pass the IDs via XCom. + from airflow.providers.anthropic.hooks.anthropic import AnthropicHook + + hook = AnthropicHook(conn_id=ANTHROPIC_CONN_ID) + agent = hook.create_agent( + name="airflow-research-agent", + system="You are a concise research assistant.", + tools=[{"type": "agent_toolset_20260401"}], + ) + environment = hook.create_environment(name="airflow-agent-env") + return {"agent_id": agent.id, "environment_id": environment.id} + + setup = setup_agent_and_environment() + + # [START howto_operator_anthropic_agent_session] + from airflow.providers.anthropic.operators.agent import AnthropicAgentSessionOperator + + run_agent = AnthropicAgentSessionOperator( + task_id="run_agent", + conn_id=ANTHROPIC_CONN_ID, + agent_id=setup["agent_id"], + environment_id=setup["environment_id"], + message="Summarize the latest stable Apache Airflow release in two sentences.", + deferrable=True, + ) + # [END howto_operator_anthropic_agent_session] + + run_agent + + +anthropic_managed_agent() + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example Dag with pytest (see: contributing-docs/testing/system_tests.rst) +test_run = get_test_run(dag) diff --git a/providers/anthropic/tests/system/anthropic/example_anthropic_batch.py b/providers/anthropic/tests/system/anthropic/example_anthropic_batch.py new file mode 100644 index 0000000000000..aefd0c3da9800 --- /dev/null +++ b/providers/anthropic/tests/system/anthropic/example_anthropic_batch.py @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import Any + +from airflow.sdk import dag, task + +ANTHROPIC_CONN_ID = "anthropic_default" +MODEL = "claude-opus-4-8" + +POKEMON = ["pikachu", "charmander", "bulbasaur"] + + +@dag(schedule=None, catchup=False) +def anthropic_batch_messages(): + @task + def build_requests(names: list[str]) -> list[dict[str, Any]]: + return [ + { + "custom_id": name, + "params": { + "model": MODEL, + "max_tokens": 256, + "messages": [{"role": "user", "content": f"Describe {name} in one sentence."}], + }, + } + for name in names + ] + + @task + def collect_results(batch_id: str) -> dict[str, str]: + # Results stream from the API unordered; key them by custom_id. For large + # batches, persist to object storage instead of returning via XCom. + from airflow.providers.anthropic.hooks.anthropic import AnthropicHook + + hook = AnthropicHook(conn_id=ANTHROPIC_CONN_ID) + summaries: dict[str, str] = {} + for entry in hook.stream_batch_results(batch_id): + if entry.result.type == "succeeded": + text = next((b.text for b in entry.result.message.content if b.type == "text"), "") + summaries[entry.custom_id] = text + return summaries + + requests = build_requests(POKEMON) + + # [START howto_operator_anthropic_batch] + from airflow.providers.anthropic.operators.anthropic import AnthropicBatchOperator + + run_batch = AnthropicBatchOperator( + task_id="run_batch", + conn_id=ANTHROPIC_CONN_ID, + requests=requests, + deferrable=True, + ) + # [END howto_operator_anthropic_batch] + + collect_results(batch_id=run_batch.output) + + +anthropic_batch_messages() + + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example Dag with pytest (see: contributing-docs/testing/system_tests.rst) +test_run = get_test_run(dag) diff --git a/providers/anthropic/tests/unit/__init__.py b/providers/anthropic/tests/unit/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/anthropic/tests/unit/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/anthropic/tests/unit/anthropic/__init__.py b/providers/anthropic/tests/unit/anthropic/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/anthropic/tests/unit/anthropic/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/anthropic/tests/unit/anthropic/hooks/__init__.py b/providers/anthropic/tests/unit/anthropic/hooks/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/anthropic/tests/unit/anthropic/hooks/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/anthropic/tests/unit/anthropic/hooks/test_anthropic.py b/providers/anthropic/tests/unit/anthropic/hooks/test_anthropic.py new file mode 100644 index 0000000000000..809239e5c7bf2 --- /dev/null +++ b/providers/anthropic/tests/unit/anthropic/hooks/test_anthropic.py @@ -0,0 +1,464 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.providers.anthropic.exceptions import ( + AnthropicAgentSessionError, + AnthropicAgentSessionTimeout, + AnthropicBatchTimeout, + AnthropicError, +) +from airflow.providers.anthropic.hooks.anthropic import ( + DEFAULT_MODEL, + AnthropicHook, + BatchStatus, + SessionStatus, + evaluate_session_state, +) + +pytest.importorskip("anthropic") + +HOOK_PATH = "airflow.providers.anthropic.hooks.anthropic" + + +def _conn(password="sk-ant-test", host=None, extra=None): + conn = mock.MagicMock() + conn.password = password + conn.host = host + conn.extra_dejson = extra or {} + return conn + + +def _make_hook(extra=None): + """Build a hook with the cached connection + client pre-injected (no real lookup).""" + hook = AnthropicHook() + hook.__dict__["_connection"] = _conn(extra=extra) + hook.__dict__["platform"] = (extra or {}).get("platform", "anthropic").lower() + client = mock.MagicMock() + hook.__dict__["conn"] = client + return hook, client + + +class TestBatchStatus: + @pytest.mark.parametrize( + ("status", "expected"), + [("in_progress", True), ("canceling", True), ("ended", False)], + ) + def test_is_in_progress(self, status, expected): + assert BatchStatus.is_in_progress(status) is expected + + +class TestSessionStatus: + @pytest.mark.parametrize( + ("status", "expected"), + [("running", False), ("rescheduling", False), ("idle", True), ("terminated", True)], + ) + def test_is_terminal(self, status, expected): + assert SessionStatus.is_terminal(status) is expected + + +def _session(status, outcome_results=None): + s = mock.MagicMock() + s.status = status + s.id = "sess_1" + s.outcome_evaluations = [mock.MagicMock(result=r) for r in (outcome_results or [])] + return s + + +def _idle_event(reason, event_id="evt_idle"): + return mock.MagicMock(type="session.status_idle", id=event_id, stop_reason=mock.MagicMock(type=reason)) + + +class TestEvaluateSessionState: + def test_terminated_is_done_with_error(self): + done, err, check = evaluate_session_state(_session("terminated"), expect_outcome=False) + assert done is True + assert err is not None + assert "terminated" in err + assert check is False + + @pytest.mark.parametrize("status", ["running", "rescheduling"]) + def test_non_idle_not_done(self, status): + assert evaluate_session_state(_session(status), expect_outcome=False) == (False, None, False) + + def test_message_idle_needs_event_check(self): + # message run: the session object can't say why it's idle -> defer to the event log + assert evaluate_session_state(_session("idle"), expect_outcome=False) == (False, None, True) + + def test_outcome_idle_without_verdict_not_done(self): + # just-created idle: no verdict yet -> keep waiting (fixes the start race) + assert evaluate_session_state(_session("idle"), expect_outcome=True) == (False, None, False) + + def test_outcome_satisfied_is_done_success(self): + assert evaluate_session_state(_session("idle", ["satisfied"]), expect_outcome=True) == ( + True, + None, + False, + ) + + @pytest.mark.parametrize("result", ["failed", "max_iterations_reached", "interrupted"]) + def test_outcome_failure_is_done_with_error(self, result): + done, err, check = evaluate_session_state(_session("idle", [result]), expect_outcome=True) + assert done is True + assert err is not None + assert result in err + + +class TestPollSessionCompletion: + def test_terminated_is_error(self): + hook, client = _make_hook() + client.beta.sessions.retrieve.return_value = _session("terminated") + done, err = hook.poll_session_completion("s") + assert done is True + assert err is not None + + def test_message_end_turn_success(self): + hook, client = _make_hook() + client.beta.sessions.retrieve.return_value = _session("idle") + client.beta.sessions.events.list.return_value = [_idle_event("end_turn")] + assert hook.poll_session_completion("s", kickoff_event_id="evt_kick") == (True, None) + + @pytest.mark.parametrize("reason", ["requires_action", "retries_exhausted"]) + def test_message_blocked_is_error(self, reason): + hook, client = _make_hook() + client.beta.sessions.retrieve.return_value = _session("idle") + client.beta.sessions.events.list.return_value = [_idle_event(reason)] + done, err = hook.poll_session_completion("s", kickoff_event_id="evt_kick") + assert done is True + assert err is not None + assert reason in err + + def test_message_no_response_yet_not_done(self): + # newest event is our kickoff (agent hasn't responded) -> keep waiting (start race) + hook, client = _make_hook() + client.beta.sessions.retrieve.return_value = _session("idle") + client.beta.sessions.events.list.return_value = [mock.MagicMock(type="user.message", id="evt_kick")] + assert hook.poll_session_completion("s", kickoff_event_id="evt_kick") == (False, None) + + def test_outcome_satisfied_skips_event_check(self): + hook, client = _make_hook() + client.beta.sessions.retrieve.return_value = _session("idle", ["satisfied"]) + assert hook.poll_session_completion("s", expect_outcome=True) == (True, None) + client.beta.sessions.events.list.assert_not_called() + + +class TestDefaultModel: + def test_defaults_to_constant(self): + hook, _ = _make_hook() + assert hook.default_model == DEFAULT_MODEL + + def test_from_connection_extra(self): + hook, _ = _make_hook(extra={"model": "claude-sonnet-4-6"}) + assert hook.default_model == "claude-sonnet-4-6" + + def test_create_message_uses_connection_model(self): + hook, client = _make_hook(extra={"model": "claude-sonnet-4-6"}) + hook.create_message([{"role": "user", "content": "hi"}]) + assert client.messages.create.call_args.kwargs["model"] == "claude-sonnet-4-6" + + def test_explicit_model_overrides_connection(self): + hook, client = _make_hook(extra={"model": "claude-sonnet-4-6"}) + hook.create_message([{"role": "user", "content": "hi"}], model="claude-haiku-4-5") + assert client.messages.create.call_args.kwargs["model"] == "claude-haiku-4-5" + + def test_create_agent_uses_connection_model(self): + hook, client = _make_hook(extra={"model": "claude-sonnet-4-6"}) + hook.create_agent(name="a") + assert client.beta.agents.create.call_args.kwargs["model"] == "claude-sonnet-4-6" + + +class TestManagedAgentsHook: + def _hook_with_client(self, extra=None): + hook = AnthropicHook() + hook.__dict__["_connection"] = _conn(extra=extra) + hook.__dict__["platform"] = (extra or {}).get("platform", "anthropic").lower() + client = mock.MagicMock() + hook.__dict__["conn"] = client + return hook, client + + def test_create_agent(self): + hook, client = self._hook_with_client() + hook.create_agent(name="ag1", system="be brief") + client.beta.agents.create.assert_called_once_with(name="ag1", model=DEFAULT_MODEL, system="be brief") + + def test_archive_session(self): + hook, client = self._hook_with_client() + hook.archive_session("sess_1") + client.beta.sessions.archive.assert_called_once_with("sess_1") + + def test_create_session(self): + hook, client = self._hook_with_client() + hook.create_session(agent="ag", environment_id="env") + client.beta.sessions.create.assert_called_once_with(agent="ag", environment_id="env") + + def test_create_environment_defaults_cloud(self): + hook, client = self._hook_with_client() + hook.create_environment(name="env1") + client.beta.environments.create.assert_called_once_with( + name="env1", config={"type": "cloud", "networking": {"type": "unrestricted"}} + ) + + def test_send_event_wraps_in_events_list(self): + hook, client = self._hook_with_client() + event = {"type": "user.message", "content": [{"type": "text", "text": "hi"}]} + hook.send_event("sess_1", event) + client.beta.sessions.events.send.assert_called_once_with("sess_1", events=[event]) + + def test_get_session(self): + hook, client = self._hook_with_client() + hook.get_session("sess_1") + client.beta.sessions.retrieve.assert_called_once_with("sess_1") + + @pytest.mark.parametrize("platform", ["bedrock", "vertex", "foundry"]) + def test_managed_agents_unavailable_on_non_first_party(self, platform): + hook, _ = self._hook_with_client(extra={"platform": platform}) + with pytest.raises(AnthropicError, match="Managed Agents is not available"): + hook.create_session(agent="ag", environment_id="env") + + +class TestWaitForSession: + @mock.patch(f"{HOOK_PATH}.time.sleep") + @mock.patch.object(AnthropicHook, "poll_session_completion") + @mock.patch.object(AnthropicHook, "get_connection") + def test_returns_when_done(self, mock_get_connection, mock_poll, mock_sleep): + mock_get_connection.return_value = _conn() + mock_poll.side_effect = [(False, None), (True, None)] + AnthropicHook().wait_for_session("sess_1", poll_interval=0.01) + assert mock_poll.call_count == 2 + + @mock.patch(f"{HOOK_PATH}.time.monotonic") + @mock.patch(f"{HOOK_PATH}.time.sleep") + @mock.patch.object(AnthropicHook, "poll_session_completion") + @mock.patch.object(AnthropicHook, "get_connection") + def test_raises_on_timeout(self, mock_get_connection, mock_poll, mock_sleep, mock_monotonic): + mock_get_connection.return_value = _conn() + mock_poll.return_value = (False, None) + mock_monotonic.side_effect = [0, 100] + with pytest.raises(AnthropicAgentSessionTimeout, match="did not reach a terminal status"): + AnthropicHook().wait_for_session("sess_1", poll_interval=0.01, timeout=10) + + @mock.patch(f"{HOOK_PATH}.time.sleep") + @mock.patch.object(AnthropicHook, "poll_session_completion") + @mock.patch.object(AnthropicHook, "get_connection") + def test_failure_raises(self, mock_get_connection, mock_poll, mock_sleep): + mock_get_connection.return_value = _conn() + mock_poll.return_value = (True, "Outcome not satisfied for session sess_1: failed.") + with pytest.raises(AnthropicAgentSessionError, match="not satisfied"): + AnthropicHook().wait_for_session("sess_1", expect_outcome=True, poll_interval=0.01) + + +class TestAnthropicHookGetConn: + @mock.patch(f"{HOOK_PATH}.Anthropic") + @mock.patch.object(AnthropicHook, "get_connection") + def test_first_party_uses_password_and_host(self, mock_get_connection, mock_anthropic): + mock_get_connection.return_value = _conn(password="sk-ant", host="https://gw.example") + client = AnthropicHook().get_conn() + mock_anthropic.assert_called_once_with(api_key="sk-ant", base_url="https://gw.example") + assert client is mock_anthropic.return_value + + @mock.patch(f"{HOOK_PATH}.Anthropic") + @mock.patch.object(AnthropicHook, "get_connection") + def test_client_kwargs_override_api_key_and_base_url(self, mock_get_connection, mock_anthropic): + mock_get_connection.return_value = _conn( + password="from-password", + extra={"anthropic_client_kwargs": {"api_key": "from-extra", "max_retries": 5}}, + ) + AnthropicHook().get_conn() + mock_anthropic.assert_called_once_with(api_key="from-extra", base_url=None, max_retries=5) + + @mock.patch(f"{HOOK_PATH}.AnthropicBedrock") + @mock.patch.object(AnthropicHook, "get_connection") + def test_bedrock_platform(self, mock_get_connection, mock_bedrock): + mock_get_connection.return_value = _conn(extra={"platform": "bedrock", "aws_region": "us-east-1"}) + client = AnthropicHook().get_conn() + mock_bedrock.assert_called_once_with(aws_region="us-east-1") + assert client is mock_bedrock.return_value + + @mock.patch(f"{HOOK_PATH}.AnthropicVertex") + @mock.patch.object(AnthropicHook, "get_connection") + def test_vertex_platform(self, mock_get_connection, mock_vertex): + mock_get_connection.return_value = _conn( + extra={"platform": "vertex", "project_id": "p1", "region": "us-central1"} + ) + AnthropicHook().get_conn() + mock_vertex.assert_called_once_with(project_id="p1", region="us-central1") + + @mock.patch(f"{HOOK_PATH}.AnthropicAWS") + @mock.patch.object(AnthropicHook, "get_connection") + def test_aws_platform(self, mock_get_connection, mock_aws): + mock_get_connection.return_value = _conn(extra={"platform": "aws"}) + AnthropicHook().get_conn() + mock_aws.assert_called_once_with() + + @mock.patch(f"{HOOK_PATH}.AnthropicFoundry") + @mock.patch.object(AnthropicHook, "get_connection") + def test_foundry_platform(self, mock_get_connection, mock_foundry): + mock_get_connection.return_value = _conn( + password="azkey", extra={"platform": "foundry", "resource": "r1"} + ) + AnthropicHook().get_conn() + mock_foundry.assert_called_once_with(api_key="azkey", resource="r1") + + @mock.patch.object(AnthropicHook, "get_connection") + def test_unknown_platform_raises(self, mock_get_connection): + mock_get_connection.return_value = _conn(extra={"platform": "bogus"}) + with pytest.raises(AnthropicError, match="Unknown Anthropic platform"): + AnthropicHook().get_conn() + + @mock.patch(f"{HOOK_PATH}.Anthropic") + @mock.patch(f"{HOOK_PATH}.IdentityTokenFile") + @mock.patch(f"{HOOK_PATH}.WorkloadIdentityCredentials") + @mock.patch.object(AnthropicHook, "get_connection") + def test_workload_identity_federation_explicit( + self, mock_get_connection, mock_wic, mock_itf, mock_anthropic + ): + mock_get_connection.return_value = _conn( + password=None, + extra={ + "workload_identity": { + "identity_token_file": "/var/run/secrets/anthropic.com/token", + "federation_rule_id": "fdrl_x", + "organization_id": "org_x", + "service_account_id": "svac_x", + "workspace_id": "wrkspc_x", + } + }, + ) + AnthropicHook().get_conn() + mock_itf.assert_called_once_with("/var/run/secrets/anthropic.com/token") + mock_wic.assert_called_once_with( + identity_token_provider=mock_itf.return_value, + federation_rule_id="fdrl_x", + organization_id="org_x", + service_account_id="svac_x", + workspace_id="wrkspc_x", + ) + mock_anthropic.assert_called_once_with(credentials=mock_wic.return_value, base_url=None) + + @mock.patch(f"{HOOK_PATH}.Anthropic") + @mock.patch.object(AnthropicHook, "get_connection") + def test_no_key_lets_sdk_resolve_credentials(self, mock_get_connection, mock_anthropic): + # No static key and no explicit WIF config: construct without api_key so the SDK + # resolves env-driven Workload Identity Federation / profile credentials. + mock_get_connection.return_value = _conn(password=None) + AnthropicHook().get_conn() + mock_anthropic.assert_called_once_with(base_url=None) + + +class TestAnthropicHookFeatures: + def _hook_with_client(self, extra=None): + hook = AnthropicHook() + # Pre-populate the cached_property values so no real connection lookup or + # client construction happens. + hook.__dict__["_connection"] = _conn(extra=extra) + hook.__dict__["platform"] = (extra or {}).get("platform", "anthropic").lower() + client = mock.MagicMock() + hook.__dict__["conn"] = client + return hook, client + + def test_create_message_passes_params(self): + hook, client = self._hook_with_client() + hook.create_message([{"role": "user", "content": "hi"}], system="be brief") + client.messages.create.assert_called_once_with( + model=DEFAULT_MODEL, + max_tokens=1024, + messages=[{"role": "user", "content": "hi"}], + system="be brief", + ) + + def test_count_tokens_returns_input_tokens(self): + hook, client = self._hook_with_client() + client.messages.count_tokens.return_value.input_tokens = 42 + assert hook.count_tokens([{"role": "user", "content": "hi"}]) == 42 + + def test_create_batch_passes_requests(self): + hook, client = self._hook_with_client() + reqs = [{"custom_id": "a", "params": {"model": DEFAULT_MODEL, "max_tokens": 1, "messages": []}}] + hook.create_batch(reqs) + client.messages.batches.create.assert_called_once_with(requests=reqs) + + def test_get_and_cancel_batch(self): + hook, client = self._hook_with_client() + hook.get_batch("batch_1") + client.messages.batches.retrieve.assert_called_once_with("batch_1") + hook.cancel_batch("batch_1") + client.messages.batches.cancel.assert_called_once_with("batch_1") + + def test_stream_batch_results_is_generator(self): + hook, client = self._hook_with_client() + client.messages.batches.results.return_value = iter([1, 2, 3]) + assert list(hook.stream_batch_results("batch_1")) == [1, 2, 3] + + @pytest.mark.parametrize("platform", ["bedrock", "vertex", "foundry"]) + def test_batch_features_unavailable_on_non_first_party(self, platform): + hook, _ = self._hook_with_client(extra={"platform": platform}) + with pytest.raises(AnthropicError, match="not available on the"): + hook.create_batch([]) + with pytest.raises(AnthropicError, match="not available on the"): + hook.count_tokens([{"role": "user", "content": "x"}]) + + +class TestWaitForBatch: + def _batch(self, status): + batch = mock.MagicMock() + batch.processing_status = status + return batch + + @mock.patch(f"{HOOK_PATH}.time.sleep") + @mock.patch.object(AnthropicHook, "get_batch") + @mock.patch.object(AnthropicHook, "get_connection") + def test_returns_when_ended(self, mock_get_connection, mock_get_batch, mock_sleep): + mock_get_connection.return_value = _conn() + mock_get_batch.side_effect = [self._batch("in_progress"), self._batch("ended")] + AnthropicHook().wait_for_batch("batch_1", wait_seconds=0.01) + assert mock_get_batch.call_count == 2 + + @mock.patch(f"{HOOK_PATH}.time.monotonic") + @mock.patch(f"{HOOK_PATH}.time.sleep") + @mock.patch.object(AnthropicHook, "get_batch") + @mock.patch.object(AnthropicHook, "get_connection") + def test_raises_on_timeout(self, mock_get_connection, mock_get_batch, mock_sleep, mock_monotonic): + mock_get_connection.return_value = _conn() + mock_get_batch.return_value = self._batch("in_progress") + mock_monotonic.side_effect = [0, 100] # start=0, elapsed=100 > timeout + with pytest.raises(AnthropicBatchTimeout, match="did not reach a terminal status"): + AnthropicHook().wait_for_batch("batch_1", wait_seconds=0.01, timeout=10) + + +class TestTestConnection: + @mock.patch.object(AnthropicHook, "get_connection") + def test_first_party_calls_models_list(self, mock_get_connection): + mock_get_connection.return_value = _conn() + hook = AnthropicHook() + client = mock.MagicMock() + hook.__dict__["conn"] = client + ok, msg = hook.test_connection() + assert ok is True + client.models.list.assert_called_once() + + @mock.patch.object(AnthropicHook, "get_conn") + @mock.patch.object(AnthropicHook, "get_connection") + def test_non_first_party_skips_live_check(self, mock_get_connection, mock_get_conn): + mock_get_connection.return_value = _conn(extra={"platform": "bedrock", "aws_region": "us-east-1"}) + ok, msg = AnthropicHook().test_connection() + assert ok is True + assert "no live check" in msg + mock_get_conn.assert_called_once() diff --git a/providers/anthropic/tests/unit/anthropic/operators/__init__.py b/providers/anthropic/tests/unit/anthropic/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/anthropic/tests/unit/anthropic/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/anthropic/tests/unit/anthropic/operators/test_agent.py b/providers/anthropic/tests/unit/anthropic/operators/test_agent.py new file mode 100644 index 0000000000000..f9dae030671f3 --- /dev/null +++ b/providers/anthropic/tests/unit/anthropic/operators/test_agent.py @@ -0,0 +1,179 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.exceptions import TaskDeferred +from airflow.providers.anthropic.exceptions import AnthropicAgentSessionError, AnthropicAgentSessionTimeout +from airflow.providers.anthropic.hooks.anthropic import AnthropicHook +from airflow.providers.anthropic.operators.agent import AnthropicAgentSessionOperator +from airflow.providers.anthropic.triggers.agent import AnthropicAgentSessionTrigger + +pytest.importorskip("anthropic") + + +def _context(): + return {"ti": mock.MagicMock()} + + +def test_requires_exactly_one_of_message_or_outcome(): + with pytest.raises(ValueError, match="exactly one"): + AnthropicAgentSessionOperator(task_id="a", agent_id="ag", environment_id="env") + with pytest.raises(ValueError, match="exactly one"): + AnthropicAgentSessionOperator( + task_id="a", agent_id="ag", environment_id="env", message="hi", outcome={"description": "x"} + ) + + +def test_outcome_requires_rubric(): + with pytest.raises(ValueError, match="rubric"): + AnthropicAgentSessionOperator( + task_id="a", agent_id="ag", environment_id="env", outcome={"description": "x"} + ) + + +class TestExecute: + @mock.patch.object(AnthropicAgentSessionOperator, "hook", new_callable=mock.PropertyMock) + def test_message_sends_user_message_and_waits(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + hook.create_session.return_value.id = "sess_1" + mock_hook_prop.return_value = hook + + op = AnthropicAgentSessionOperator( + task_id="a", agent_id="ag", environment_id="env", message="summarize", deferrable=False + ) + context = _context() + assert op.execute(context) == "sess_1" + hook.create_session.assert_called_once_with(agent="ag", environment_id="env") + hook.send_event.assert_called_once_with( + "sess_1", {"type": "user.message", "content": [{"type": "text", "text": "summarize"}]} + ) + hook.wait_for_session.assert_called_once() + context["ti"].xcom_push.assert_called_once_with(key="session_id", value="sess_1") + + @mock.patch.object(AnthropicAgentSessionOperator, "hook", new_callable=mock.PropertyMock) + def test_outcome_sends_define_outcome(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + hook.create_session.return_value.id = "sess_1" + mock_hook_prop.return_value = hook + + outcome = {"description": "build a CSV", "rubric": {"type": "text", "content": "has a price column"}} + op = AnthropicAgentSessionOperator( + task_id="a", agent_id="ag", environment_id="env", outcome=outcome, deferrable=False + ) + op.execute(_context()) + hook.send_event.assert_called_once_with("sess_1", {"type": "user.define_outcome", **outcome}) + + @mock.patch.object(AnthropicAgentSessionOperator, "hook", new_callable=mock.PropertyMock) + def test_passes_vault_ids_and_resources(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + hook.create_session.return_value.id = "sess_1" + mock_hook_prop.return_value = hook + + op = AnthropicAgentSessionOperator( + task_id="a", + agent_id="ag", + environment_id="env", + message="hi", + deferrable=False, + vault_ids=["vlt_1"], + session_resources=[{"type": "file", "file_id": "f1", "mount_path": "/workspace/f"}], + ) + op.execute(_context()) + hook.create_session.assert_called_once_with( + agent="ag", + environment_id="env", + vault_ids=["vlt_1"], + resources=[{"type": "file", "file_id": "f1", "mount_path": "/workspace/f"}], + ) + + @mock.patch.object(AnthropicAgentSessionOperator, "hook", new_callable=mock.PropertyMock) + def test_sync_timeout_archives_and_raises(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + hook.create_session.return_value.id = "sess_1" + hook.wait_for_session.side_effect = AnthropicAgentSessionTimeout("too slow") + mock_hook_prop.return_value = hook + + op = AnthropicAgentSessionOperator( + task_id="a", agent_id="ag", environment_id="env", message="hi", deferrable=False + ) + with pytest.raises(AnthropicAgentSessionTimeout, match="too slow"): + op.execute(_context()) + hook.archive_session.assert_called_once_with("sess_1") + + @mock.patch.object(AnthropicAgentSessionOperator, "hook", new_callable=mock.PropertyMock) + def test_deferrable_defers_with_trigger(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + hook.create_session.return_value.id = "sess_1" + mock_hook_prop.return_value = hook + + op = AnthropicAgentSessionOperator( + task_id="a", agent_id="ag", environment_id="env", message="hi", deferrable=True + ) + with pytest.raises(TaskDeferred) as exc: + op.execute(_context()) + assert isinstance(exc.value.trigger, AnthropicAgentSessionTrigger) + assert exc.value.trigger.session_id == "sess_1" + assert exc.value.method_name == "execute_complete" + hook.wait_for_session.assert_not_called() + + +class TestExecuteComplete: + def test_success_returns_session_id(self): + op = AnthropicAgentSessionOperator(task_id="a", agent_id="ag", environment_id="env", message="hi") + assert op.execute_complete({}, {"status": "success", "session_id": "sess_1"}) == "sess_1" + + def test_error_raises(self): + op = AnthropicAgentSessionOperator(task_id="a", agent_id="ag", environment_id="env", message="hi") + with pytest.raises(AnthropicAgentSessionError, match="boom"): + op.execute_complete({}, {"status": "error", "session_id": "s", "message": "boom"}) + + @mock.patch.object(AnthropicAgentSessionOperator, "hook", new_callable=mock.PropertyMock) + def test_timeout_archives_and_raises(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + mock_hook_prop.return_value = hook + op = AnthropicAgentSessionOperator(task_id="a", agent_id="ag", environment_id="env", message="hi") + with pytest.raises(AnthropicAgentSessionTimeout): + op.execute_complete({}, {"status": "timeout", "session_id": "s", "message": "slow"}) + hook.archive_session.assert_called_once_with("s") + + def test_none_event_raises(self): + op = AnthropicAgentSessionOperator(task_id="a", agent_id="ag", environment_id="env", message="hi") + with pytest.raises(AnthropicAgentSessionError, match="without an event"): + op.execute_complete({}, None) + + +class TestOnKill: + @mock.patch.object(AnthropicAgentSessionOperator, "hook", new_callable=mock.PropertyMock) + def test_on_kill_archives_session(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + mock_hook_prop.return_value = hook + op = AnthropicAgentSessionOperator(task_id="a", agent_id="ag", environment_id="env", message="hi") + op.session_id = "sess_1" + op.on_kill() + hook.archive_session.assert_called_once_with("sess_1") + + @mock.patch.object(AnthropicAgentSessionOperator, "hook", new_callable=mock.PropertyMock) + def test_on_kill_noop_without_session(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + mock_hook_prop.return_value = hook + op = AnthropicAgentSessionOperator(task_id="a", agent_id="ag", environment_id="env", message="hi") + op.on_kill() + hook.archive_session.assert_not_called() diff --git a/providers/anthropic/tests/unit/anthropic/operators/test_anthropic.py b/providers/anthropic/tests/unit/anthropic/operators/test_anthropic.py new file mode 100644 index 0000000000000..f78ae682b62fc --- /dev/null +++ b/providers/anthropic/tests/unit/anthropic/operators/test_anthropic.py @@ -0,0 +1,190 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.exceptions import TaskDeferred +from airflow.providers.anthropic.exceptions import AnthropicBatchJobError, AnthropicBatchTimeout +from airflow.providers.anthropic.hooks.anthropic import AnthropicHook +from airflow.providers.anthropic.operators.anthropic import AnthropicBatchOperator +from airflow.providers.anthropic.triggers.anthropic import AnthropicBatchTrigger +from airflow.providers.common.compat.sdk import AirflowSkipException + +pytest.importorskip("anthropic") + +REQUESTS = [{"custom_id": "a", "params": {"model": "claude-opus-4-8", "max_tokens": 8, "messages": []}}] + + +def _counts(succeeded=0, errored=0, canceled=0, expired=0, processing=0): + counts = mock.MagicMock() + counts.succeeded = succeeded + counts.errored = errored + counts.canceled = canceled + counts.expired = expired + counts.processing = processing + return counts + + +def _context(): + return {"ti": mock.MagicMock()} + + +class TestAnthropicBatchOperatorExecute: + @mock.patch.object(AnthropicBatchOperator, "hook", new_callable=mock.PropertyMock) + def test_sync_waits_and_returns_batch_id(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + hook.create_batch.return_value.id = "batch_1" + hook.wait_for_batch.return_value.request_counts = _counts(succeeded=1) + mock_hook_prop.return_value = hook + + op = AnthropicBatchOperator(task_id="t", requests=REQUESTS, deferrable=False) + context = _context() + result = op.execute(context) + + assert result == "batch_1" + hook.wait_for_batch.assert_called_once() + context["ti"].xcom_push.assert_called_once_with(key="batch_id", value="batch_1") + + @mock.patch.object(AnthropicBatchOperator, "hook", new_callable=mock.PropertyMock) + def test_deferrable_defers_with_trigger(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + hook.create_batch.return_value.id = "batch_1" + mock_hook_prop.return_value = hook + + op = AnthropicBatchOperator(task_id="t", requests=REQUESTS, deferrable=True) + with pytest.raises(TaskDeferred) as exc: + op.execute(_context()) + assert isinstance(exc.value.trigger, AnthropicBatchTrigger) + assert exc.value.trigger.batch_id == "batch_1" + assert exc.value.method_name == "execute_complete" + hook.wait_for_batch.assert_not_called() + + @mock.patch.object(AnthropicBatchOperator, "hook", new_callable=mock.PropertyMock) + def test_no_wait_returns_immediately(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + hook.create_batch.return_value.id = "batch_1" + mock_hook_prop.return_value = hook + + op = AnthropicBatchOperator(task_id="t", requests=REQUESTS, wait_for_completion=False) + assert op.execute(_context()) == "batch_1" + hook.wait_for_batch.assert_not_called() + + @mock.patch.object(AnthropicBatchOperator, "hook", new_callable=mock.PropertyMock) + def test_sync_applies_policy_skip_on_full_cancel(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + hook.create_batch.return_value.id = "batch_1" + hook.wait_for_batch.return_value.request_counts = _counts(canceled=2) + mock_hook_prop.return_value = hook + + op = AnthropicBatchOperator(task_id="t", requests=REQUESTS, deferrable=False) + with pytest.raises(AirflowSkipException): + op.execute(_context()) + + @mock.patch.object(AnthropicBatchOperator, "hook", new_callable=mock.PropertyMock) + def test_sync_applies_policy_fail_on_partial_error_when_strict(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + hook.create_batch.return_value.id = "batch_1" + hook.wait_for_batch.return_value.request_counts = _counts(succeeded=4, errored=1) + mock_hook_prop.return_value = hook + + op = AnthropicBatchOperator( + task_id="t", requests=REQUESTS, deferrable=False, fail_on_partial_error=True + ) + with pytest.raises(AnthropicBatchJobError): + op.execute(_context()) + + @mock.patch.object(AnthropicBatchOperator, "hook", new_callable=mock.PropertyMock) + def test_sync_timeout_cancels_and_raises(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + hook.create_batch.return_value.id = "batch_1" + hook.wait_for_batch.side_effect = AnthropicBatchTimeout("too slow") + mock_hook_prop.return_value = hook + + op = AnthropicBatchOperator(task_id="t", requests=REQUESTS, deferrable=False) + with pytest.raises(AnthropicBatchTimeout, match="too slow"): + op.execute(_context()) + hook.cancel_batch.assert_called_once_with("batch_1") + + @mock.patch.object(AnthropicBatchOperator, "hook", new_callable=mock.PropertyMock) + def test_empty_requests_raises_before_any_api_call(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + mock_hook_prop.return_value = hook + + op = AnthropicBatchOperator(task_id="t", requests=[]) + with pytest.raises(ValueError, match="at least one request"): + op.execute(_context()) + hook.create_batch.assert_not_called() + + +class TestExecuteComplete: + def test_success_returns_batch_id(self): + op = AnthropicBatchOperator(task_id="t", requests=REQUESTS) + event = {"status": "success", "batch_id": "batch_1", "request_counts": {"succeeded": 3}} + assert op.execute_complete(_context(), event) == "batch_1" + + def test_error_raises_job_error(self): + op = AnthropicBatchOperator(task_id="t", requests=REQUESTS) + event = {"status": "error", "batch_id": "batch_1", "message": "boom"} + with pytest.raises(AnthropicBatchJobError, match="boom"): + op.execute_complete(_context(), event) + + @mock.patch("airflow.providers.anthropic.operators.anthropic.AnthropicHook") + def test_timeout_cancels_and_raises(self, mock_hook_cls): + op = AnthropicBatchOperator(task_id="t", requests=REQUESTS) + event = {"status": "timeout", "batch_id": "batch_1", "message": "too slow"} + with pytest.raises(AnthropicBatchTimeout, match="too slow"): + op.execute_complete(_context(), event) + mock_hook_cls.return_value.cancel_batch.assert_called_once_with("batch_1") + + def test_fully_cancelled_skips(self): + op = AnthropicBatchOperator(task_id="t", requests=REQUESTS) + event = {"status": "success", "batch_id": "batch_1", "request_counts": {"canceled": 4}} + with pytest.raises(AirflowSkipException): + op.execute_complete(_context(), event) + + def test_partial_error_warns_by_default(self): + op = AnthropicBatchOperator(task_id="t", requests=REQUESTS) + event = {"status": "success", "batch_id": "b", "request_counts": {"succeeded": 9, "errored": 1}} + assert op.execute_complete(_context(), event) == "b" + + def test_partial_error_fails_when_strict(self): + op = AnthropicBatchOperator(task_id="t", requests=REQUESTS, fail_on_partial_error=True) + event = {"status": "success", "batch_id": "b", "request_counts": {"succeeded": 9, "errored": 1}} + with pytest.raises(AnthropicBatchJobError, match="failed request"): + op.execute_complete(_context(), event) + + +class TestOnKill: + @mock.patch.object(AnthropicBatchOperator, "hook", new_callable=mock.PropertyMock) + def test_on_kill_cancels_known_batch(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + mock_hook_prop.return_value = hook + op = AnthropicBatchOperator(task_id="t", requests=REQUESTS) + op.batch_id = "batch_1" + op.on_kill() + hook.cancel_batch.assert_called_once_with("batch_1") + + @mock.patch.object(AnthropicBatchOperator, "hook", new_callable=mock.PropertyMock) + def test_on_kill_noop_without_batch(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + mock_hook_prop.return_value = hook + op = AnthropicBatchOperator(task_id="t", requests=REQUESTS) + op.on_kill() + hook.cancel_batch.assert_not_called() diff --git a/providers/anthropic/tests/unit/anthropic/sensors/__init__.py b/providers/anthropic/tests/unit/anthropic/sensors/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/anthropic/tests/unit/anthropic/sensors/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/anthropic/tests/unit/anthropic/sensors/test_anthropic.py b/providers/anthropic/tests/unit/anthropic/sensors/test_anthropic.py new file mode 100644 index 0000000000000..644ed1a29ee23 --- /dev/null +++ b/providers/anthropic/tests/unit/anthropic/sensors/test_anthropic.py @@ -0,0 +1,104 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.exceptions import TaskDeferred +from airflow.providers.anthropic.exceptions import AnthropicBatchJobError, AnthropicBatchTimeout +from airflow.providers.anthropic.hooks.anthropic import AnthropicHook +from airflow.providers.anthropic.sensors.anthropic import AnthropicBatchSensor +from airflow.providers.anthropic.triggers.anthropic import AnthropicBatchTrigger +from airflow.providers.common.compat.sdk import AirflowSkipException + +pytest.importorskip("anthropic") + + +def _batch(status, succeeded=0, errored=0, canceled=0, expired=0): + batch = mock.MagicMock() + batch.processing_status = status + counts = batch.request_counts + counts.succeeded, counts.errored, counts.canceled, counts.expired = ( + succeeded, + errored, + canceled, + expired, + ) + return batch + + +class TestAnthropicBatchSensorPoke: + @mock.patch.object(AnthropicBatchSensor, "hook", new_callable=mock.PropertyMock) + def test_poke_false_while_in_progress(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + hook.get_batch.return_value = _batch("in_progress") + mock_hook_prop.return_value = hook + sensor = AnthropicBatchSensor(task_id="s", batch_id="b1") + assert sensor.poke({}) is False + + @mock.patch.object(AnthropicBatchSensor, "hook", new_callable=mock.PropertyMock) + def test_poke_true_when_ended(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + hook.get_batch.return_value = _batch("ended", succeeded=5) + mock_hook_prop.return_value = hook + sensor = AnthropicBatchSensor(task_id="s", batch_id="b1") + assert sensor.poke({}) is True + + @mock.patch.object(AnthropicBatchSensor, "hook", new_callable=mock.PropertyMock) + def test_poke_skips_on_full_cancel(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + hook.get_batch.return_value = _batch("ended", canceled=3) + mock_hook_prop.return_value = hook + sensor = AnthropicBatchSensor(task_id="s", batch_id="b1") + with pytest.raises(AirflowSkipException): + sensor.poke({}) + + @mock.patch.object(AnthropicBatchSensor, "hook", new_callable=mock.PropertyMock) + def test_poke_fails_on_partial_error_when_strict(self, mock_hook_prop): + hook = mock.MagicMock(spec=AnthropicHook) + hook.get_batch.return_value = _batch("ended", succeeded=4, errored=1) + mock_hook_prop.return_value = hook + sensor = AnthropicBatchSensor(task_id="s", batch_id="b1", fail_on_partial_error=True) + with pytest.raises(AnthropicBatchJobError): + sensor.poke({}) + + +class TestAnthropicBatchSensorDeferrable: + def test_execute_defers(self): + sensor = AnthropicBatchSensor(task_id="s", batch_id="b1", deferrable=True) + with pytest.raises(TaskDeferred) as exc: + sensor.execute({}) + assert isinstance(exc.value.trigger, AnthropicBatchTrigger) + assert exc.value.trigger.batch_id == "b1" + assert exc.value.method_name == "execute_complete" + + def test_execute_complete_error_raises(self): + sensor = AnthropicBatchSensor(task_id="s", batch_id="b1") + with pytest.raises(AnthropicBatchJobError, match="boom"): + sensor.execute_complete({}, {"status": "error", "batch_id": "b1", "message": "boom"}) + + def test_execute_complete_timeout_raises(self): + sensor = AnthropicBatchSensor(task_id="s", batch_id="b1") + with pytest.raises(AnthropicBatchTimeout): + sensor.execute_complete({}, {"status": "timeout", "batch_id": "b1", "message": "slow"}) + + def test_execute_complete_success(self): + sensor = AnthropicBatchSensor(task_id="s", batch_id="b1") + event = {"status": "success", "batch_id": "b1", "request_counts": {"succeeded": 2}} + assert sensor.execute_complete({}, event) is None diff --git a/providers/anthropic/tests/unit/anthropic/test_exceptions.py b/providers/anthropic/tests/unit/anthropic/test_exceptions.py new file mode 100644 index 0000000000000..365ba875eb4cb --- /dev/null +++ b/providers/anthropic/tests/unit/anthropic/test_exceptions.py @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +from airflow.providers.anthropic.exceptions import ( + AnthropicAgentSessionError, + AnthropicAgentSessionTimeout, + AnthropicBatchJobError, + AnthropicBatchTimeout, + AnthropicError, +) + + +@pytest.mark.parametrize( + "exc", + [ + AnthropicError, + AnthropicBatchJobError, + AnthropicBatchTimeout, + AnthropicAgentSessionError, + AnthropicAgentSessionTimeout, + ], +) +def test_provider_errors_share_base_and_are_not_airflow_exceptions(exc): + # Every provider error is catchable via the AnthropicError base and is deliberately + # NOT an AirflowException subclass (the no-new-AirflowException direction). + assert issubclass(exc, AnthropicError) + assert not any(base.__name__ == "AirflowException" for base in exc.__mro__) diff --git a/providers/anthropic/tests/unit/anthropic/triggers/__init__.py b/providers/anthropic/tests/unit/anthropic/triggers/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/anthropic/tests/unit/anthropic/triggers/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/anthropic/tests/unit/anthropic/triggers/test_agent.py b/providers/anthropic/tests/unit/anthropic/triggers/test_agent.py new file mode 100644 index 0000000000000..1da3822be26f2 --- /dev/null +++ b/providers/anthropic/tests/unit/anthropic/triggers/test_agent.py @@ -0,0 +1,120 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import time +from unittest import mock + +import pytest + +from airflow.providers.anthropic.triggers.agent import AnthropicAgentSessionTrigger +from airflow.triggers.base import TriggerEvent + +pytest.importorskip("anthropic") + +TRIGGER_PATH = "airflow.providers.anthropic.triggers.agent" +POLL = "airflow.providers.anthropic.hooks.anthropic.AnthropicHook.poll_session_completion" + + +def _trigger(end_time=None, expect_outcome=False): + return AnthropicAgentSessionTrigger( + conn_id="anthropic_default", + session_id="sess_1", + poll_interval=1.0, + end_time=end_time if end_time is not None else time.time() + 3600, + expect_outcome=expect_outcome, + kickoff_event_id="evt_kick", + ) + + +def test_serialization(): + end_time = time.time() + 3600 + path, kwargs = _trigger(end_time).serialize() + assert path == "airflow.providers.anthropic.triggers.agent.AnthropicAgentSessionTrigger" + assert kwargs == { + "conn_id": "anthropic_default", + "session_id": "sess_1", + "poll_interval": 1.0, + "end_time": end_time, + "expect_outcome": False, + "kickoff_event_id": "evt_kick", + } + + +@pytest.mark.asyncio +@mock.patch(POLL) +async def test_done_success_yields_success(mock_poll): + mock_poll.return_value = (True, None) + event = await _trigger().run().__anext__() + assert event.payload["status"] == "success" + assert event.payload["session_id"] == "sess_1" + + +@pytest.mark.asyncio +@mock.patch(POLL) +async def test_done_error_yields_error(mock_poll): + mock_poll.return_value = (True, "Session sess_1 terminated.") + event = await _trigger().run().__anext__() + assert event.payload["status"] == "error" + assert "terminated" in event.payload["message"] + + +@pytest.mark.asyncio +@mock.patch(POLL) +async def test_timeout_yields_timeout(mock_poll): + mock_poll.return_value = (False, None) + event = await _trigger(end_time=time.time() - 1).run().__anext__() + assert event.payload["status"] == "timeout" + + +@pytest.mark.asyncio +@mock.patch(f"{TRIGGER_PATH}.asyncio.sleep") +@mock.patch(POLL) +async def test_polls_until_done(mock_poll, mock_sleep): + mock_poll.side_effect = [(False, None), (False, None), (True, None)] + event = await _trigger().run().__anext__() + assert event.payload["status"] == "success" + assert mock_poll.call_count == 3 + assert mock_sleep.await_count == 2 + + +@pytest.mark.asyncio +@mock.patch(f"{TRIGGER_PATH}.asyncio.sleep") +@mock.patch(POLL) +async def test_persistent_error_yields_error_after_retries(mock_poll, mock_sleep): + mock_poll.side_effect = RuntimeError("kaboom") + event = await _trigger().run().__anext__() + assert event == TriggerEvent({"status": "error", "session_id": "sess_1", "message": "kaboom"}) + assert mock_poll.call_count == 5 + + +@pytest.mark.asyncio +@mock.patch(f"{TRIGGER_PATH}.asyncio.sleep") +@mock.patch(POLL) +async def test_transient_error_then_success(mock_poll, mock_sleep): + mock_poll.side_effect = [RuntimeError("blip"), (True, None)] + event = await _trigger().run().__anext__() + assert event.payload["status"] == "success" + + +@pytest.mark.asyncio +@mock.patch(POLL) +async def test_outcome_failure_yields_error(mock_poll): + mock_poll.return_value = (True, "Outcome not satisfied for session sess_1: max_iterations_reached.") + event = await _trigger(expect_outcome=True).run().__anext__() + assert event.payload["status"] == "error" + assert "max_iterations_reached" in event.payload["message"] diff --git a/providers/anthropic/tests/unit/anthropic/triggers/test_anthropic.py b/providers/anthropic/tests/unit/anthropic/triggers/test_anthropic.py new file mode 100644 index 0000000000000..21f12941f9876 --- /dev/null +++ b/providers/anthropic/tests/unit/anthropic/triggers/test_anthropic.py @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import time +from unittest import mock + +import pytest + +from airflow.providers.anthropic.triggers.anthropic import AnthropicBatchTrigger +from airflow.triggers.base import TriggerEvent + +pytest.importorskip("anthropic") + +TRIGGER_PATH = "airflow.providers.anthropic.triggers.anthropic" +HOOK_PATH = "airflow.providers.anthropic.hooks.anthropic.AnthropicHook.get_batch" + + +def _batch(status, succeeded=0, errored=0, canceled=0, expired=0, processing=0): + batch = mock.MagicMock() + batch.processing_status = status + counts = mock.MagicMock() + counts.succeeded, counts.errored = succeeded, errored + counts.canceled, counts.expired, counts.processing = canceled, expired, processing + batch.request_counts = counts + return batch + + +class TestAnthropicBatchTrigger: + CONN_ID = "anthropic_default" + BATCH_ID = "batch_1" + POLL = 1.0 + + def _trigger(self, end_time=None): + return AnthropicBatchTrigger( + conn_id=self.CONN_ID, + batch_id=self.BATCH_ID, + poll_interval=self.POLL, + end_time=end_time if end_time is not None else time.time() + 3600, + ) + + def test_serialization(self): + end_time = time.time() + 3600 + class_path, kwargs = self._trigger(end_time).serialize() + assert class_path == "airflow.providers.anthropic.triggers.anthropic.AnthropicBatchTrigger" + assert kwargs == { + "conn_id": self.CONN_ID, + "batch_id": self.BATCH_ID, + "poll_interval": self.POLL, + "end_time": end_time, + } + + @pytest.mark.asyncio + @mock.patch(HOOK_PATH) + async def test_ended_yields_success_with_counts(self, mock_get_batch): + mock_get_batch.return_value = _batch("ended", succeeded=3, errored=1) + event = await self._trigger().run().__anext__() + assert event == TriggerEvent( + { + "status": "success", + "batch_id": self.BATCH_ID, + "message": f"Batch {self.BATCH_ID} has ended.", + "request_counts": { + "succeeded": 3, + "errored": 1, + "canceled": 0, + "expired": 0, + "processing": 0, + }, + } + ) + + @pytest.mark.asyncio + @mock.patch(HOOK_PATH) + async def test_timeout_yields_timeout_event(self, mock_get_batch): + mock_get_batch.return_value = _batch("in_progress") + # end_time already in the past -> timeout on first poll. + event = await self._trigger(end_time=time.time() - 1).run().__anext__() + assert event.payload["status"] == "timeout" + assert event.payload["batch_id"] == self.BATCH_ID + + @pytest.mark.asyncio + @mock.patch(f"{TRIGGER_PATH}.asyncio.sleep") + @mock.patch(HOOK_PATH) + async def test_persistent_error_yields_error_after_retries(self, mock_get_batch, mock_sleep): + mock_get_batch.side_effect = RuntimeError("kaboom") + event = await self._trigger().run().__anext__() + assert event == TriggerEvent({"status": "error", "batch_id": self.BATCH_ID, "message": "kaboom"}) + assert mock_get_batch.call_count == 5 + + @pytest.mark.asyncio + @mock.patch(f"{TRIGGER_PATH}.asyncio.sleep") + @mock.patch(HOOK_PATH) + async def test_transient_error_then_success(self, mock_get_batch, mock_sleep): + mock_get_batch.side_effect = [RuntimeError("blip"), _batch("ended", succeeded=1)] + event = await self._trigger().run().__anext__() + assert event.payload["status"] == "success" + + @pytest.mark.asyncio + @mock.patch(f"{TRIGGER_PATH}.asyncio.sleep") + @mock.patch(HOOK_PATH) + async def test_polls_until_ended(self, mock_get_batch, mock_sleep): + mock_get_batch.side_effect = [ + _batch("in_progress"), + _batch("canceling"), + _batch("ended", succeeded=1), + ] + event = await self._trigger().run().__anext__() + assert event.payload["status"] == "success" + assert mock_get_batch.call_count == 3 + assert mock_sleep.await_count == 2 diff --git a/pyproject.toml b/pyproject.toml index 36975fb70ac6e..79caae5d340af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,6 +110,9 @@ apache-airflow = "airflow.__main__:main" "amazon" = [ "apache-airflow-providers-amazon>=9.0.0" ] +"anthropic" = [ + "apache-airflow-providers-anthropic>=0.1.0" +] "apache.cassandra" = [ "apache-airflow-providers-apache-cassandra>=3.7.0; python_version !=\"3.14\"" ] @@ -411,6 +414,7 @@ apache-airflow = "airflow.__main__:main" "apache-airflow-providers-akeyless>=0.1.0", "apache-airflow-providers-alibaba>=3.0.0", "apache-airflow-providers-amazon>=9.0.0", + "apache-airflow-providers-anthropic>=0.1.0", "apache-airflow-providers-apache-cassandra>=3.7.0; python_version !=\"3.14\"", "apache-airflow-providers-apache-drill>=2.8.1", "apache-airflow-providers-apache-druid>=3.12.0", @@ -1090,6 +1094,8 @@ mypy_path = [ "$MYPY_CONFIG_FILE_DIR/providers/alibaba/tests", "$MYPY_CONFIG_FILE_DIR/providers/amazon/src", "$MYPY_CONFIG_FILE_DIR/providers/amazon/tests", + "$MYPY_CONFIG_FILE_DIR/providers/anthropic/src", + "$MYPY_CONFIG_FILE_DIR/providers/anthropic/tests", "$MYPY_CONFIG_FILE_DIR/providers/apache/cassandra/src", "$MYPY_CONFIG_FILE_DIR/providers/apache/cassandra/tests", "$MYPY_CONFIG_FILE_DIR/providers/apache/drill/src", @@ -1427,6 +1433,7 @@ apache-airflow-providers-airbyte = false apache-airflow-providers-akeyless = false apache-airflow-providers-alibaba = false apache-airflow-providers-amazon = false +apache-airflow-providers-anthropic = false apache-airflow-providers-apache-cassandra = false apache-airflow-providers-apache-drill = false apache-airflow-providers-apache-druid = false @@ -1571,6 +1578,7 @@ apache-airflow-providers-airbyte = false apache-airflow-providers-akeyless = false apache-airflow-providers-alibaba = false apache-airflow-providers-amazon = false +apache-airflow-providers-anthropic = false apache-airflow-providers-apache-cassandra = false apache-airflow-providers-apache-drill = false apache-airflow-providers-apache-druid = false @@ -1730,6 +1738,7 @@ apache-airflow-providers-airbyte = { workspace = true } apache-airflow-providers-akeyless = { workspace = true } apache-airflow-providers-alibaba = { workspace = true } apache-airflow-providers-amazon = { workspace = true } +apache-airflow-providers-anthropic = { workspace = true } apache-airflow-providers-apache-cassandra = { workspace = true } apache-airflow-providers-apache-drill = { workspace = true } apache-airflow-providers-apache-druid = { workspace = true } @@ -1869,6 +1878,7 @@ members = [ "providers/akeyless", "providers/alibaba", "providers/amazon", + "providers/anthropic", "providers/apache/cassandra", "providers/apache/drill", "providers/apache/druid", diff --git a/scripts/ci/docker-compose/remove-sources.yml b/scripts/ci/docker-compose/remove-sources.yml index f10ae1472e9d6..ed2e4b1bed789 100644 --- a/scripts/ci/docker-compose/remove-sources.yml +++ b/scripts/ci/docker-compose/remove-sources.yml @@ -30,6 +30,7 @@ services: - ../../../empty:/opt/airflow/providers/akeyless/src - ../../../empty:/opt/airflow/providers/alibaba/src - ../../../empty:/opt/airflow/providers/amazon/src + - ../../../empty:/opt/airflow/providers/anthropic/src - ../../../empty:/opt/airflow/providers/apache/beam/src - ../../../empty:/opt/airflow/providers/apache/cassandra/src - ../../../empty:/opt/airflow/providers/apache/drill/src diff --git a/scripts/ci/docker-compose/tests-sources.yml b/scripts/ci/docker-compose/tests-sources.yml index d33deafc970fa..1f6e47ba961c2 100644 --- a/scripts/ci/docker-compose/tests-sources.yml +++ b/scripts/ci/docker-compose/tests-sources.yml @@ -43,6 +43,7 @@ services: - ../../../providers/akeyless/tests:/opt/airflow/providers/akeyless/tests - ../../../providers/alibaba/tests:/opt/airflow/providers/alibaba/tests - ../../../providers/amazon/tests:/opt/airflow/providers/amazon/tests + - ../../../providers/anthropic/tests:/opt/airflow/providers/anthropic/tests - ../../../providers/apache/beam/tests:/opt/airflow/providers/apache/beam/tests - ../../../providers/apache/cassandra/tests:/opt/airflow/providers/apache/cassandra/tests - ../../../providers/apache/drill/tests:/opt/airflow/providers/apache/drill/tests diff --git a/uv.lock b/uv.lock index 709e8ee883e4d..b2a5a709622f4 100644 --- a/uv.lock +++ b/uv.lock @@ -68,6 +68,7 @@ apache-airflow-providers-microsoft-mssql = false apache-airflow-providers-teradata = false apache-airflow-providers-jdbc = false apache-airflow-helm-chart = false +apache-airflow-providers-anthropic = false apache-airflow-providers-common-io = false apache-airflow-providers-cohere = false apache-airflow-providers-pinecone = false @@ -175,6 +176,7 @@ members = [ "apache-airflow-providers-akeyless", "apache-airflow-providers-alibaba", "apache-airflow-providers-amazon", + "apache-airflow-providers-anthropic", "apache-airflow-providers-apache-cassandra", "apache-airflow-providers-apache-drill", "apache-airflow-providers-apache-druid", @@ -952,6 +954,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/f2/bee5de8a2699fc8a3cce34d61c7a2626a2c310ddde7ea5611327eb0ddbe9/anthropic-0.109.2-py3-none-any.whl", hash = "sha256:e0fb4ca5df0ed983248c9c6c3242adc81d9cfddb8725902da53698554117abac", size = 923800, upload-time = "2026-06-15T17:30:23.124Z" }, ] +[package.optional-dependencies] +aws = [ + { name = "boto3" }, + { name = "botocore" }, +] +bedrock = [ + { name = "boto3" }, + { name = "botocore" }, +] +vertex = [ + { name = "google-auth", extra = ["requests"] }, +] + [[package]] name = "anyio" version = "4.14.0" @@ -995,6 +1010,7 @@ all = [ { name = "apache-airflow-providers-akeyless" }, { name = "apache-airflow-providers-alibaba" }, { name = "apache-airflow-providers-amazon", extra = ["aiobotocore", "python3-saml", "s3fs"] }, + { name = "apache-airflow-providers-anthropic" }, { name = "apache-airflow-providers-apache-cassandra", marker = "python_full_version != '3.14.*'" }, { name = "apache-airflow-providers-apache-drill" }, { name = "apache-airflow-providers-apache-druid" }, @@ -1113,6 +1129,9 @@ amazon = [ amazon-aws-auth = [ { name = "apache-airflow-providers-amazon", extra = ["python3-saml"] }, ] +anthropic = [ + { name = "apache-airflow-providers-anthropic" }, +] apache-atlas = [ { name = "atlasclient" }, ] @@ -1561,6 +1580,8 @@ requires-dist = [ { name = "apache-airflow-providers-amazon", extras = ["aiobotocore"], marker = "extra == 'aiobotocore'", editable = "providers/amazon" }, { name = "apache-airflow-providers-amazon", extras = ["python3-saml"], marker = "extra == 'amazon-aws-auth'", editable = "providers/amazon" }, { name = "apache-airflow-providers-amazon", extras = ["s3fs"], marker = "extra == 's3fs'", editable = "providers/amazon" }, + { name = "apache-airflow-providers-anthropic", marker = "extra == 'all'", editable = "providers/anthropic" }, + { name = "apache-airflow-providers-anthropic", marker = "extra == 'anthropic'", editable = "providers/anthropic" }, { name = "apache-airflow-providers-apache-cassandra", marker = "python_full_version != '3.14.*' and extra == 'all'", editable = "providers/apache/cassandra" }, { name = "apache-airflow-providers-apache-cassandra", marker = "python_full_version != '3.14.*' and extra == 'apache-cassandra'", editable = "providers/apache/cassandra" }, { name = "apache-airflow-providers-apache-drill", marker = "extra == 'all'", editable = "providers/apache/drill" }, @@ -1772,7 +1793,7 @@ requires-dist = [ { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.30.0" }, { name = "uv", marker = "extra == 'uv'", specifier = ">=0.11.21" }, ] -provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] +provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] [package.metadata.requires-dev] ci-image = [ @@ -3213,6 +3234,58 @@ dev = [ ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] +[[package]] +name = "apache-airflow-providers-anthropic" +version = "0.1.0" +source = { editable = "providers/anthropic" } +dependencies = [ + { name = "anthropic" }, + { name = "apache-airflow" }, + { name = "apache-airflow-providers-common-compat" }, +] + +[package.optional-dependencies] +aws = [ + { name = "anthropic", extra = ["aws"] }, +] +bedrock = [ + { name = "anthropic", extra = ["bedrock"] }, +] +vertex = [ + { name = "anthropic", extra = ["vertex"] }, +] + +[package.dev-dependencies] +dev = [ + { name = "apache-airflow" }, + { name = "apache-airflow-devel-common" }, + { name = "apache-airflow-providers-common-compat" }, + { name = "apache-airflow-task-sdk" }, +] +docs = [ + { name = "apache-airflow-devel-common", extra = ["docs"] }, +] + +[package.metadata] +requires-dist = [ + { name = "anthropic", specifier = ">=0.101.0" }, + { name = "anthropic", extras = ["aws"], marker = "extra == 'aws'", specifier = ">=0.101.0" }, + { name = "anthropic", extras = ["bedrock"], marker = "extra == 'bedrock'", specifier = ">=0.101.0" }, + { name = "anthropic", extras = ["vertex"], marker = "extra == 'vertex'", specifier = ">=0.101.0" }, + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, +] +provides-extras = ["bedrock", "vertex", "aws"] + +[package.metadata.requires-dev] +dev = [ + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-devel-common", editable = "devel-common" }, + { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, + { name = "apache-airflow-task-sdk", editable = "task-sdk" }, +] +docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] + [[package]] name = "apache-airflow-providers-apache-cassandra" version = "3.9.5"