diff --git a/examples/ai-agent/ai-agent-capability-catalog.yaml b/examples/ai-agent/ai-agent-capability-catalog.yaml new file mode 100644 index 00000000..34197072 --- /dev/null +++ b/examples/ai-agent/ai-agent-capability-catalog.yaml @@ -0,0 +1,143 @@ +metadata: + id: ATR-AI-AGENT-CAPABILITY-CATALOG + type: CapabilityCatalog + gemara-version: "1.1.0" + version: "0.1.0" + description: > + Capabilities of AI agent systems that create the opportunity for the + threats catalogued by Agent Threat Rules (ATR). Each capability describes + a feature or function an agent can perform, not a threat. Threats reference + these capabilities; ATR detection rule categories map to them in a companion + MappingDocument. Authored by the ATR project and validated against the + Gemara schemas; ATR hosts the content, Gemara defines the schema. + author: + id: atr + name: Adam Lin + type: Human + contact: + name: Adam Lin + affiliation: Agent Threat Rules + email: adam@agentthreatrule.org + social: eeee2345 + +title: AI Agent Capability Catalog + +groups: + - id: model-interaction + title: Model Interaction + description: > + Capabilities by which an agent receives instructions and untrusted + content into a language model and acts on the model's output. + - id: tool-and-protocol + title: Tool and Protocol Access + description: > + Capabilities by which an agent invokes external tools and connects to + tool servers over protocols such as the Model Context Protocol (MCP). + - id: extensibility + title: Extensibility + description: > + Capabilities by which an agent loads and runs third-party extensions + such as skills, plugins, and packaged capability bundles. + - id: state-and-memory + title: State and Memory + description: > + Capabilities by which an agent reads and writes persistent state, + conversation context, and long-term memory across turns and sessions. + - id: autonomy-and-coordination + title: Autonomy and Coordination + description: > + Capabilities by which an agent acts without per-step human approval and + coordinates with other agents. + +capabilities: + - id: CAP-MODEL-INFERENCE + title: Model Inference on Untrusted Input + description: > + The agent passes user instructions and content drawn from external + sources into a language model and treats the resulting output as + actionable. Mixing trusted instructions with untrusted content in a + single context window is the feature that makes prompt injection and + jailbreak attempts possible. + group: model-interaction + + - id: CAP-OUTPUT-ACTION + title: Acting on Model Output + description: > + The agent converts model output into actions, tool calls, or content + returned to downstream systems. Because output is consumed without an + independent trust boundary, manipulated output can redirect behaviour + or carry exfiltrated data. + group: model-interaction + + - id: CAP-TOOL-INVOCATION + title: Tool Invocation + description: > + The agent calls external tools and functions, passing arguments and + receiving results that re-enter the model context. Tool arguments and + tool results are an attack surface for injection, command execution, + and over-broad actions. + group: tool-and-protocol + + - id: CAP-MCP-CONNECTION + title: MCP and Tool-Server Connection + description: > + The agent connects to tool servers over the Model Context Protocol or + similar transports, consuming server-provided tool manifests, schemas, + and descriptions. Server-controlled metadata is trusted at connection + time, which creates the opportunity for tool poisoning and tool + redefinition (rug-pull) after initial approval. + group: tool-and-protocol + + - id: CAP-CROSS-AGENT-MESSAGING + title: Cross-Agent Message Passing + description: > + The agent sends and receives messages to and from other agents in a + multi-agent workflow. Messages from peer agents are treated as + trusted input, which lets a compromised or manipulated agent influence + others. + group: tool-and-protocol + + - id: CAP-SKILL-LOADING + title: Skill and Plugin Loading + description: > + The agent loads third-party skills, plugins, or capability bundles and + executes the instructions and code they contain. Loaded extensions run + with the agent's privileges, which creates the opportunity for supply + chain compromise and backdoored or over-privileged skills. + group: extensibility + + - id: CAP-MEMORY-WRITE + title: Memory and State Write + description: > + The agent writes facts, instructions, or artifacts into persistent + memory or shared state that influences later turns and sessions. + Writable memory is the feature that lets poisoned content persist and + take effect after the originating input is gone. + group: state-and-memory + + - id: CAP-MEMORY-READ + title: Context and Memory Read + description: > + The agent reads conversation history, retrieved documents, and + long-term memory back into the active context. Reading attacker- + influenced state re-introduces untrusted content and is a channel for + indirect injection and staged data exfiltration. + group: state-and-memory + + - id: CAP-AUTONOMOUS-ACTION + title: Autonomous Action Without Per-Step Approval + description: > + The agent executes multi-step plans, loops, and side-effecting actions + without human approval at each step. Unbounded autonomy is the feature + that turns a single manipulated decision into runaway loops, resource + abuse, or unauthorized real-world actions. + group: autonomy-and-coordination + + - id: CAP-PRIVILEGED-EXECUTION + title: Privileged and Delegated Execution + description: > + The agent runs with delegated credentials and access to systems, + databases, files, and networks. Holding standing privilege creates the + opportunity for privilege escalation, sandbox escape, and use of the + agent's access beyond its intended scope. + group: autonomy-and-coordination diff --git a/examples/ai-agent/atr-categories-to-capabilities-mapping.yaml b/examples/ai-agent/atr-categories-to-capabilities-mapping.yaml new file mode 100644 index 00000000..98e0c4f8 --- /dev/null +++ b/examples/ai-agent/atr-categories-to-capabilities-mapping.yaml @@ -0,0 +1,207 @@ +title: ATR Rule Categories to AI Agent Capabilities +metadata: + id: ATR-CAP-MAP-001 + version: "0.1.0" + type: MappingDocument + gemara-version: "1.1.0" + description: > + Maps Agent Threat Rules (ATR) detection rule categories to the AI agent + capabilities that create the opportunity for each category of threat. + The source artifact is the ATR rule corpus, grouped by its nine detection + categories; the target artifact is the AI Agent Capability Catalog. + Relationships are expressed as relates-to following the convention that + external cross-references are refined by downstream consumers in their + applicability context. + author: + id: atr + name: Adam Lin + type: Human + contact: + name: Adam Lin + affiliation: Agent Threat Rules + email: adam@agentthreatrule.org + social: eeee2345 + mapping-references: + - id: ATR + title: Agent Threat Rules + version: "0.1.0" + url: "https://github.com/Agent-Threat-Rule/agent-threat-rules" + description: > + Open MIT detection standard for AI agent threats. Rules are grouped + into detection categories under rules/; the reference-ids below are + those category identifiers. + - id: ATR-AI-AGENT-CAP + title: AI Agent Capability Catalog + version: "0.1.0" + description: > + Capabilities of AI agent systems that create the opportunity for the + threats ATR detects. + +source-reference: + reference-id: ATR + entry-type: Control +target-reference: + reference-id: ATR-AI-AGENT-CAP + entry-type: Capability +remarks: > + Each source entry-id is an ATR rule category. Targets are capability ids in + the AI Agent Capability Catalog. A category may relate to more than one + capability when its rules cover threats arising from several agent features. + +mappings: + - id: MAP-prompt-injection + source: prompt-injection + relationship: relates-to + targets: + - entry-id: CAP-MODEL-INFERENCE + strength: 9 + confidence-level: High + rationale: > + Prompt injection and jailbreak rules detect attacker instructions + smuggled into the model context. They exist because the agent runs + inference over a context that mixes trusted instructions with + untrusted content. + - entry-id: CAP-MEMORY-READ + strength: 6 + confidence-level: Medium + rationale: > + Indirect injection rules detect malicious instructions delivered + through retrieved content read back into context, which depends on + the agent reading external or remembered state. + + - id: MAP-tool-poisoning + source: tool-poisoning + relationship: relates-to + targets: + - entry-id: CAP-TOOL-INVOCATION + strength: 8 + confidence-level: High + rationale: > + Tool-poisoning rules detect injection and unsafe behaviour through + tool arguments and tool results, which is only reachable because the + agent invokes tools and feeds their output back into the model. + - entry-id: CAP-MCP-CONNECTION + strength: 9 + confidence-level: High + rationale: > + A large share of tool-poisoning rules target MCP tool manifests, + schemas, and server name fields, and tool redefinition after + approval (rug-pull), which depend on the agent trusting + server-provided metadata at connection time. + + - id: MAP-context-exfiltration + source: context-exfiltration + relationship: relates-to + targets: + - entry-id: CAP-OUTPUT-ACTION + strength: 8 + confidence-level: High + rationale: > + Exfiltration rules detect secrets and sensitive context leaving + through model output and downstream actions, which requires the + agent to act on and emit model output. + - entry-id: CAP-TOOL-INVOCATION + strength: 7 + confidence-level: Medium + rationale: > + Many exfiltration rules detect data carried out through tool calls + and tool responses, depending on the agent's tool invocation + capability. + - entry-id: CAP-MEMORY-READ + strength: 5 + confidence-level: Medium + rationale: > + Staged exfiltration rules detect sensitive data pulled from context + and memory before being leaked, depending on the agent reading + stored state. + + - id: MAP-agent-manipulation + source: agent-manipulation + relationship: relates-to + targets: + - entry-id: CAP-MODEL-INFERENCE + strength: 8 + confidence-level: High + rationale: > + Manipulation rules detect authority claims, persona injection, and + goal drift that steer the agent's decisions through its inference + over untrusted content. + - entry-id: CAP-CROSS-AGENT-MESSAGING + strength: 6 + confidence-level: Medium + rationale: > + Some manipulation rules detect influence delivered between agents, + which depends on the agent trusting peer-agent messages. + + - id: MAP-privilege-escalation + source: privilege-escalation + relationship: relates-to + targets: + - entry-id: CAP-PRIVILEGED-EXECUTION + strength: 9 + confidence-level: High + rationale: > + Privilege-escalation rules detect sandbox escape, stacked SQL DML + abuse, and delayed-execution bypass, which are reachable only + because the agent runs with standing, delegated privilege. + + - id: MAP-excessive-autonomy + source: excessive-autonomy + relationship: relates-to + targets: + - entry-id: CAP-AUTONOMOUS-ACTION + strength: 9 + confidence-level: High + rationale: > + Excessive-autonomy rules detect runaway tool-call loops, SSRF via + autonomous fetches, and unauthorized orchestration, which depend on + the agent acting across steps without per-step approval. + + - id: MAP-data-poisoning + source: data-poisoning + relationship: relates-to + targets: + - entry-id: CAP-MEMORY-WRITE + strength: 9 + confidence-level: High + rationale: > + Data-poisoning rules detect persistent memory plants and poisoned + stored facts, which depend on the agent writing attacker-influenced + content into durable memory or state. + - entry-id: CAP-MEMORY-READ + strength: 6 + confidence-level: Medium + rationale: > + Poisoned data causes harm when it is read back into a later turn, + depending on the agent's context and memory read capability. + + - id: MAP-model-abuse + source: model-abuse + relationship: relates-to + targets: + - entry-id: CAP-MODEL-INFERENCE + strength: 7 + confidence-level: Medium + rationale: > + Model-abuse rules detect attempts to misuse the model to produce + harmful or fraudulent output, which arises from the agent running + inference and treating the result as a deliverable. + - entry-id: CAP-OUTPUT-ACTION + strength: 6 + confidence-level: Medium + rationale: > + Harm in this category lands through the produced and delivered + output, depending on the agent acting on model output. + + - id: MAP-skill-compromise + source: skill-compromise + relationship: relates-to + targets: + - entry-id: CAP-SKILL-LOADING + strength: 9 + confidence-level: High + rationale: > + Skill-compromise rules detect backdoored, impersonated, and + over-privileged skills, which can only execute because the agent + loads and runs third-party skills and plugins with its own + privileges. diff --git a/test/schema_test.go b/test/schema_test.go index 6259e4fa..68c8c0a3 100644 --- a/test/schema_test.go +++ b/test/schema_test.go @@ -69,6 +69,10 @@ func TestSchemaValidation(t *testing.T) { {"valid capability catalog", "./test-data/good-capability-catalog.yaml", "#CapabilityCatalog", false, ""}, {"vector mapping", "./test-data/good-vector-owasp-mapping.yaml", "#MappingDocument", false, ""}, + // AI agent capability catalog and ATR mappings (authored by ATR, validated against Gemara) + {"valid AI agent capability catalog", "../examples/ai-agent/ai-agent-capability-catalog.yaml", "#CapabilityCatalog", false, ""}, + {"valid ATR categories to capabilities mapping", "../examples/ai-agent/atr-categories-to-capabilities-mapping.yaml", "#MappingDocument", false, ""}, + // RiskCatalog — positive {"valid risk catalog", "./test-data/good-risk-catalog.yaml", "#RiskCatalog", false, ""},