datacache for to data intensive tools and ambient agents

contrib/crewai/tests/test_structured_tool.py

@@ -51,7 +51,7 @@ def test_structured_tool_auto_name_description():
 def test_structured_tool_validation_errors():
     # Test missing docstring
     with pytest.raises(
-        ValueError, match="Function must have a docstring if description not provided."
+        ValueError, match=r"Function must have a docstring if description not provided\."
     ):
         StructuredTool.from_function(func=unnamed_function, args_schema=CalculatorInput)
 

examples/mcp_servers/custom_server_with_prebuilt_tools/pyproject.toml

@@ -41,7 +41,9 @@ disallow_untyped_defs = false
 
 # # Uncomment the following if you are developing inside of the arcade-mcp repo & want to use editable mode
 # # Otherwise, you will install the following packages from PyPI
-# [tool.uv.sources]
-# arcade-mcp = { path = "../../../", editable = true }
-# arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
-# arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }
+[tool.uv.sources]
+arcade-mcp = { path = "../../../", editable = true }
+arcade-core = { path = "../../../libs/arcade-core/", editable = true }
+arcade-tdk = { path = "../../../libs/arcade-tdk/", editable = true }
+arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
+arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }

examples/mcp_servers/datacache/README.md

@@ -0,0 +1,50 @@
+## Datacache (DuckDB) Example Server
+
+This example demonstrates the `@app.tool(datacache={keys:[...], ttl:...})` feature and `context.datacache.*` helpers.
+
+### What it does
+- Uses a DuckDB file as a per-tool, per-identity cache
+- Downloads the DuckDB file from S3 before tool execution
+- Uploads it back to S3 after tool execution
+- Uses a Redis lock to ensure only one tool execution per cache key runs at a time
+
+### Required environment variables
+- `ARCADE_DATACACHE_REDIS_URL` (locking; still required for local storage)
+
+### Local backend (default for this example)
+This example defaults `ARCADE_DATACACHE_STORAGE_BACKEND=local` in code, so you don’t need to set it explicitly unless you want S3.
+
+Set:
+- `ARCADE_DATACACHE_REDIS_URL` (e.g. `redis://localhost:6379/0`)
+
+Optional:
+- `ARCADE_DATACACHE_LOCAL_DIR` (default: `/tmp/arcade_datacache`)
+
+### S3 backend (how to switch)
+Set:
+- `ARCADE_DATACACHE_STORAGE_BACKEND=s3`
+- `ARCADE_DATACACHE_REDIS_URL`
+- `ARCADE_DATACACHE_S3_BUCKET`
+
+Optional:
+- `ARCADE_DATACACHE_S3_PREFIX` (default: `arcade/datacache`)
+- `ARCADE_DATACACHE_AWS_REGION`
+- `ARCADE_DATACACHE_S3_ENDPOINT_URL` (e.g. MinIO)
+- `ARCADE_DATACACHE_AWS_ACCESS_KEY_ID`, `ARCADE_DATACACHE_AWS_SECRET_ACCESS_KEY`, `ARCADE_DATACACHE_AWS_SESSION_TOKEN`
+
+### Running (http by default)
+From this directory:
+
+```bash
+uv sync
+uv run python -m datacache.server
+```
+
+### Calling tools with `_meta`
+The datacache keys `organization` and `project` are read from the request `_meta` and propagated into `ToolContext.metadata`.
+
+Your MCP client must include:
+- `_meta.organization`
+- `_meta.project`
+
+`user_id` comes from the server’s normal `ToolContext.user_id` behavior.

examples/mcp_servers/datacache/pyproject.toml

@@ -0,0 +1,29 @@
+[project]
+name = "datacache"
+version = "0.1.0"
+description = "Example MCP server demonstrating Arcade datacache (DuckDB + S3 + Redis locking)"
+requires-python = ">=3.10"
+dependencies = [
+    "arcade-mcp-server>=1.9.1,<2.0.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/datacache"]
+
+[project.entry-points.arcade_toolkits]
+toolkit_name = "datacache"
+
+[tool.ruff]
+line-length = 100
+target-version = "py312"
+
+[tool.uv.sources]
+arcade-mcp = { path = "../../../", editable = true }
+arcade-core = { path = "../../../libs/arcade-core/", editable = true }
+arcade-tdk = { path = "../../../libs/arcade-tdk/", editable = true }
+arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
+arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }

examples/mcp_servers/datacache/src/datacache/__init__.py

@@ -0,0 +1 @@
+"""Datacache DuckDB example server package."""

examples/mcp_servers/datacache/src/datacache/server.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Datacache MCP server example."""
+
+import os
+import sys
+from typing import Annotated
+
+from arcade_mcp_server import Context, MCPApp
+
+# ---------------------------------------------------------------------------
+# How to set organization/project for datacache keying
+#
+# Datacache keys `organization` and `project` come from the MCP `tools/call` request
+# params `_meta`, and are propagated into `ToolContext.metadata` automatically.
+#
+# Example JSON-RPC call:
+# {
+#   "jsonrpc": "2.0",
+#   "id": 1,
+#   "method": "tools/call",
+#   "params": {
+#     "name": "datacache_upsert_profile",
+#     "arguments": { "profile_id": "1", "name": "Alice" },
+#     "_meta": { "organization": "acme", "project": "rocket" }
+#   }
+# }
+# ---------------------------------------------------------------------------
+
+# Default the example to local storage so it's easy to run locally.
+# (The MCP server requires ARCADE_DATACACHE_STORAGE_BACKEND when datacache is enabled.)
+os.environ.setdefault("ARCADE_DATACACHE_STORAGE_BACKEND", "local")
+os.environ.setdefault("ARCADE_DATACACHE_REDIS_URL", "redis://localhost:6379/0")
+
+app = MCPApp(name="datacache_duckdb", version="1.0.0", log_level="DEBUG")
+
+
+@app.tool(datacache={"keys": ["organization", "project", "user_id"], "ttl": 3600})
+async def upsert_profile(
+    context: Context,
+    profile_id: Annotated[str, "Unique profile id"],
+    name: Annotated[str, "Display name"],
+) -> dict:
+    """Upsert a profile row into a DuckDB-backed datacache."""
+    profile = {"id": profile_id, "name": name, "kind": "example_profile"}
+
+    response = await context.datacache.set(
+        "profiles",
+        profile,
+        id_col="id",
+    )
+    return response.model_dump(mode="json")
+
+
+@app.tool(datacache={"keys": ["organization", "project", "user_id"], "ttl": 3600})
+async def search_profiles(
+    context: Context,
+    term: Annotated[str, "Search term (case-insensitive substring match)"],
+) -> list[dict]:
+    """Search profiles by name using a LIKE query under the hood."""
+    return await context.datacache.search("profiles", "name", term)
+
+
+if __name__ == "__main__":
+    transport = sys.argv[1] if len(sys.argv) > 1 else "http"
+    app.run(transport=transport, host="127.0.0.1", port=8000)

examples/mcp_servers/echo/pyproject.toml

@@ -21,7 +21,9 @@ packages = ["src/echo"]
 
 # # Uncomment the following if you are developing inside of the arcade-mcp repo & want to use editable mode
 # # Otherwise, you will install the following packages from PyPI
-# [tool.uv.sources]
-# arcade-mcp = { path = "../../../", editable = true }
-# arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
-# arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }
+[tool.uv.sources]
+arcade-mcp = { path = "../../../", editable = true }
+arcade-core = { path = "../../../libs/arcade-core/", editable = true }
+arcade-tdk = { path = "../../../libs/arcade-tdk/", editable = true }
+arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
+arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }

examples/mcp_servers/local_filesystem/pyproject.toml

@@ -38,7 +38,9 @@ toolkit_name = "simple"
 
 # # Uncomment the following if you are developing inside of the arcade-mcp repo & want to use editable mode
 # # Otherwise, you will install the following packages from PyPI
-# [tool.uv.sources]
-# arcade-mcp = { path = "../../../", editable = true }
-# arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
-# arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }
+[tool.uv.sources]
+arcade-mcp = { path = "../../../", editable = true }
+arcade-core = { path = "../../../libs/arcade-core/", editable = true }
+arcade-tdk = { path = "../../../libs/arcade-tdk/", editable = true }
+arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
+arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }

examples/mcp_servers/logging/pyproject.toml

@@ -21,7 +21,9 @@ packages = ["src/logging"]
 
 # # Uncomment the following if you are developing inside of the arcade-mcp repo & want to use editable mode
 # # Otherwise, you will install the following packages from PyPI
-# [tool.uv.sources]
-# arcade-mcp = { path = "../../../", editable = true }
-# arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
-# arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }
+[tool.uv.sources]
+arcade-mcp = { path = "../../../", editable = true }
+arcade-core = { path = "../../../libs/arcade-core/", editable = true }
+arcade-tdk = { path = "../../../libs/arcade-tdk/", editable = true }
+arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
+arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }

examples/mcp_servers/progress_reporting/pyproject.toml

@@ -39,7 +39,9 @@ disallow_untyped_defs = false
 
 # # Uncomment the following if you are developing inside of the arcade-mcp repo & want to use editable mode
 # # Otherwise, you will install the following packages from PyPI
-# [tool.uv.sources]
-# arcade-mcp = { path = "../../../", editable = true }
-# arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
-# arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }
+[tool.uv.sources]
+arcade-mcp = { path = "../../../", editable = true }
+arcade-core = { path = "../../../libs/arcade-core/", editable = true }
+arcade-tdk = { path = "../../../libs/arcade-tdk/", editable = true }
+arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
+arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }

examples/mcp_servers/sampling/pyproject.toml

@@ -39,7 +39,9 @@ disallow_untyped_defs = false
 
 # # Uncomment the following if you are developing inside of the arcade-mcp repo & want to use editable mode
 # # Otherwise, you will install the following packages from PyPI
-# [tool.uv.sources]
-# arcade-mcp = { path = "../../../", editable = true }
-# arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
-# arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }
+[tool.uv.sources]
+arcade-mcp = { path = "../../../", editable = true }
+arcade-core = { path = "../../../libs/arcade-core/", editable = true }
+arcade-tdk = { path = "../../../libs/arcade-tdk/", editable = true }
+arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
+arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }

examples/mcp_servers/server_with_evaluations/pyproject.toml

@@ -40,7 +40,9 @@ disallow_untyped_defs = false
 
 # # Uncomment the following if you are developing inside of the arcade-mcp repo & want to use editable mode
 # # Otherwise, you will install the following packages from PyPI
-# [tool.uv.sources]
-# arcade-mcp = { path = "../../../", editable = true }
-# arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
-# arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }
+[tool.uv.sources]
+arcade-mcp = { path = "../../../", editable = true }
+arcade-core = { path = "../../../libs/arcade-core/", editable = true }
+arcade-tdk = { path = "../../../libs/arcade-tdk/", editable = true }
+arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
+arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }

examples/mcp_servers/simple/pyproject.toml

@@ -39,7 +39,9 @@ disallow_untyped_defs = false
 
 # # Uncomment the following if you are developing inside of the arcade-mcp repo & want to use editable mode
 # # Otherwise, you will install the following packages from PyPI
-# [tool.uv.sources]
-# arcade-mcp = { path = "../../../", editable = true }
-# arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
-# arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }
+[tool.uv.sources]
+arcade-mcp = { path = "../../../", editable = true }
+arcade-core = { path = "../../../libs/arcade-core/", editable = true }
+arcade-tdk = { path = "../../../libs/arcade-tdk/", editable = true }
+arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
+arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }

examples/mcp_servers/tool_chaining/pyproject.toml

@@ -39,7 +39,9 @@ disallow_untyped_defs = false
 
 # # Uncomment the following if you are developing inside of the arcade-mcp repo & want to use editable mode
 # # Otherwise, you will install the following packages from PyPI
-# [tool.uv.sources]
-# arcade-mcp = { path = "../../../", editable = true }
-# arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
-# arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }
+[tool.uv.sources]
+arcade-mcp = { path = "../../../", editable = true }
+arcade-core = { path = "../../../libs/arcade-core/", editable = true }
+arcade-tdk = { path = "../../../libs/arcade-tdk/", editable = true }
+arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
+arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }

examples/mcp_servers/user_elicitation/pyproject.toml

@@ -39,7 +39,9 @@ disallow_untyped_defs = false
 
 # # Uncomment the following if you are developing inside of the arcade-mcp repo & want to use editable mode
 # # Otherwise, you will install the following packages from PyPI
-# [tool.uv.sources]
-# arcade-mcp = { path = "../../../", editable = true }
-# arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
-# arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }
+[tool.uv.sources]
+arcade-mcp = { path = "../../../", editable = true }
+arcade-core = { path = "../../../libs/arcade-core/", editable = true }
+arcade-tdk = { path = "../../../libs/arcade-tdk/", editable = true }
+arcade-serve = { path = "../../../libs/arcade-serve/", editable = true }
+arcade-mcp-server = { path = "../../../libs/arcade-mcp-server/", editable = true }

libs/arcade-core/arcade_core/catalog.py

@@ -112,6 +112,7 @@ class ToolMeta(BaseModel):
     toolkit: str | None = None
     package: str | None = None
     path: str | None = None
+    datacache: dict[str, Any] | None = None
     date_added: datetime = Field(default_factory=datetime.now)
     date_updated: datetime = Field(default_factory=datetime.now)
 
@@ -261,6 +262,7 @@ def add_tool(
                 toolkit=toolkit_name,
                 package=toolkit.package_name if toolkit else None,
                 path=module.__file__ if module else None,
+                datacache=getattr(tool_func, "__tool_datacache__", None),
             ),
             input_model=input_model,
             output_model=output_model,

libs/arcade-mcp-server/arcade_mcp_server/context.py

@@ -37,6 +37,8 @@
     ToolContext,
 )
 
+from arcade_mcp_server.datacache.client import DatacacheClient
+from arcade_mcp_server.datacache.types import DatacacheSetResult
 from arcade_mcp_server.resource_server.base import ResourceOwner
 from arcade_mcp_server.types import (
     AudioContent,
@@ -147,6 +149,7 @@ def __init__(
         self._sampling = Sampling(self)
         self._ui = UI(self)
         self._notifications = Notifications(self)
+        self._datacache = Datacache(self)
 
     @property
     def server(self) -> Any:
@@ -292,6 +295,15 @@ def notifications(self) -> Notifications:
         """
         return self._notifications
 
+    @property
+    def datacache(self) -> Datacache:
+        """DuckDB-backed datacache for the currently executing tool.
+
+        This is only available when the tool was declared with:
+        `@app.tool(datacache={keys:[...]})`.
+        """
+        return self._datacache
+
     @property
     def request_id(self) -> str | None:
         """Get the current request ID.
@@ -689,6 +701,46 @@ def prompts(self) -> _NotificationsPrompts:
         return self._prompts
 
 
+class Datacache(_ContextComponent):
+    """Namespaced datacache API exposed to tool code as `context.datacache.*`."""
+
+    def _client(self) -> DatacacheClient:
+        client = getattr(self._ctx, "_datacache_client", None)
+        if client is None:
+            raise RuntimeError(
+                "Datacache is not enabled for this tool execution. "
+                "Enable it by setting `datacache={keys:[...]}` on the tool decorator."
+            )
+        return client
+
+    async def discover_databases(self) -> list[dict[str, Any]]:
+        return await self._client().discover_databases()
+
+    async def discover_tables(self, database: str) -> list[str]:
+        return await self._client().discover_tables(database)
+
+    async def discover_schema(self, database: str, table: str) -> list[dict[str, Any]]:
+        return await self._client().discover_schema(database, table)
+
+    async def query(self, database: str, table: str, sql: str) -> list[dict[str, Any]]:
+        return await self._client().query(database, table, sql)
+
+    async def set(
+        self,
+        table_name: str,
+        record: dict[str, Any],
+        id_col: str = "id",
+        ttl: int | None = None,
+    ) -> DatacacheSetResult:
+        return await self._client().set(table_name, record, id_col=id_col, ttl=ttl)
+
+    async def get(self, table_name: str, record_id: str) -> dict[str, Any] | None:
+        return await self._client().get(table_name, record_id)
+
+    async def search(self, table_name: str, field: str, value: str) -> list[dict[str, Any]]:
+        return await self._client().search(table_name, field, value)
+
+
 def get_current_model_context() -> Context | None:
     """Get the current model context if available."""
     return _current_model_context.get()