diff --git a/packages/cloud-agents/evals/router/assertions/routing-assertions.ts b/packages/cloud-agents/evals/router/assertions/routing-assertions.ts index 14f756a0..2a5f071f 100644 --- a/packages/cloud-agents/evals/router/assertions/routing-assertions.ts +++ b/packages/cloud-agents/evals/router/assertions/routing-assertions.ts @@ -7,6 +7,7 @@ interface RoutingResponse { workspaceValue?: string; reasoning?: string; confidence?: number; + kickoffMessage?: string | null; needsExternalLookup?: boolean; externalReference?: string | null; requestedModelId?: string | null; @@ -272,6 +273,49 @@ function requestedModelIdMatchesExpected( }; } +/** + * Asserts that kickoffMessage is present as short display text ending with a + * single period (the router prompt contract for chat started messages). + */ +function hasValidKickoffMessage(output: string): AssertionResult { + const json = extractJson(output); + if (!json) { + return { pass: false, score: 0, reason: 'Invalid JSON response' }; + } + + const kickoff = json.kickoffMessage; + if (typeof kickoff !== 'string' || kickoff.trim().length === 0) { + return { + pass: false, + score: 0, + reason: 'kickoffMessage is missing or empty', + }; + } + + const text = kickoff.trim(); + if (!text.endsWith('.')) { + return { + pass: false, + score: 0, + reason: `kickoffMessage must end with a period: ${text}`, + }; + } + + if (text.length < 8 || text.length > 200) { + return { + pass: false, + score: 0, + reason: `kickoffMessage length looks off (${text.length} chars): ${text}`, + }; + } + + return { + pass: true, + score: 1, + reason: `Valid kickoffMessage: ${text}`, + }; +} + /** * Asserts that the router did not report a model preference when the user did * not express one: requestedModelId must be the explicit `__no_model__` @@ -304,6 +348,7 @@ export { workspaceValueMatchesExpected, reasoningContainsExpected, requestedModelIdMatchesExpected, + hasValidKickoffMessage, requestedModelIdIsNull, extractJson, }; diff --git a/packages/cloud-agents/evals/router/datasets/basic.yaml b/packages/cloud-agents/evals/router/datasets/basic.yaml index 7a186af5..5683ca19 100644 --- a/packages/cloud-agents/evals/router/datasets/basic.yaml +++ b/packages/cloud-agents/evals/router/datasets/basic.yaml @@ -27,6 +27,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - description: 'Explanation task - understanding code behavior' vars: @@ -54,6 +56,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - description: 'Planning task - architecture design' vars: @@ -81,6 +85,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - description: 'Ambiguous repository case' vars: @@ -104,6 +110,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage # Should pick a specific repository (best guess), NOT all_repositories - type: javascript value: file://assertions/routing-assertions.ts:hasValidWorkspaceValue @@ -133,6 +141,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:hasValidWorkspaceValue @@ -163,6 +173,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected @@ -190,6 +202,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - description: 'Refactoring task - should route to Coder' vars: @@ -219,6 +233,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - description: 'Explicit project selection - user forces specific environment' vars: @@ -246,6 +262,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:hasValidWorkspaceValue - type: javascript @@ -276,6 +294,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected @@ -297,6 +317,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - description: 'Investigative question about UI regression - should route to Explainer with cloud app environment' vars: @@ -326,6 +348,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:hasValidWorkspaceValue - type: javascript @@ -359,6 +383,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:hasValidWorkspaceValue - type: javascript @@ -390,6 +416,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:hasValidWorkspaceValue - type: javascript @@ -422,6 +450,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:hasValidWorkspaceValue - type: javascript @@ -453,6 +483,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:hasValidWorkspaceValue - type: javascript diff --git a/packages/cloud-agents/evals/router/datasets/model-routing.yaml b/packages/cloud-agents/evals/router/datasets/model-routing.yaml index c08e1955..5c58698e 100644 --- a/packages/cloud-agents/evals/router/datasets/model-routing.yaml +++ b/packages/cloud-agents/evals/router/datasets/model-routing.yaml @@ -26,6 +26,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - type: javascript @@ -54,6 +56,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - type: javascript @@ -82,6 +86,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - type: javascript @@ -110,6 +116,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - type: javascript @@ -138,6 +146,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - type: javascript @@ -166,6 +176,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - type: javascript @@ -194,6 +206,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - type: javascript @@ -222,6 +236,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - type: javascript @@ -249,6 +265,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - type: javascript @@ -275,6 +293,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - type: javascript @@ -305,6 +325,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - type: javascript @@ -334,6 +356,8 @@ assert: - type: javascript value: file://assertions/routing-assertions.ts:isValidRoutingJson + - type: javascript + value: file://assertions/routing-assertions.ts:hasValidKickoffMessage - type: javascript value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected - type: javascript diff --git a/packages/cloud-agents/evals/router/prompts/routing.ts b/packages/cloud-agents/evals/router/prompts/routing.ts index 904284ee..ba552e15 100644 --- a/packages/cloud-agents/evals/router/prompts/routing.ts +++ b/packages/cloud-agents/evals/router/prompts/routing.ts @@ -149,7 +149,7 @@ Respond with a single JSON object containing exactly these fields: - "workspaceValue" (string): Name of the chosen environment. - "reasoning" (string): Brief explanation of your workspace decision. - "confidence" (number): Confidence in the workspace choice from 0 to 1. -- "kickoffMessage" (string or null): Full short user-facing kickoff sentence (about 8-18 words). Naturally include the chosen environment name, and naturally include the model display name when requestedModelId is a real model. Vary wording; no "Getting started on your task in…" boilerplate every time. No emojis, markdown, quotes, or mentions. +- "kickoffMessage" (string): Required short user-facing kickoff sentence (about 8-18 words) that ends with a period. Naturally include the chosen environment name, and naturally include the model display name when requestedModelId is a real model. Vary wording; no "Getting started on your task in…" boilerplate every time. No emojis, markdown, quotes, or mentions. Always provide a non-empty value for real routed tasks. - "needsExternalLookup" (boolean): Whether an external reference must be fetched before routing, per the external lookup rules. - "externalReference" (string or null): The exact external reference to fetch when needsExternalLookup is true, otherwise null. - "requestedModelId" (string or null): The model id the user explicitly requested from the Available Models list, or the literal "${NO_MODEL_MENTIONED_VALUE}" when the user does not name a model. @@ -160,7 +160,9 @@ Respond with a single JSON object containing exactly these fields: export default function generatePrompt({ vars }: PromptInput): string { const context = buildContext(vars); const availableModels = buildAvailableModelsSection(vars.extraModels); - const routingPrompt = buildWorkspaceRoutingPrompt(); + const routingPrompt = buildWorkspaceRoutingPrompt({ + includeKickoffMessage: true, + }); return `${routingPrompt} diff --git a/packages/cloud-agents/src/server/router/prompts/routing-prompt.ts b/packages/cloud-agents/src/server/router/prompts/routing-prompt.ts index 50aa7227..276b28e6 100644 --- a/packages/cloud-agents/src/server/router/prompts/routing-prompt.ts +++ b/packages/cloud-agents/src/server/router/prompts/routing-prompt.ts @@ -42,17 +42,16 @@ const EXTERNAL_LOOKUP_RULES = `**External lookup rules:** const KICKOFF_MESSAGE_SECTION = `## Kickoff Message -Also always set \`kickoffMessage\` to the full short user-facing kickoff for chat (one brief sentence). Roomote posts this text as-is, so weave the details in naturally instead of using a fixed template: +Also always set \`kickoffMessage\` to a short user-facing kickoff sentence for chat (about 8-18 words). Roomote posts this text as-is, so weave the details in naturally instead of using a fixed template. Write a complete sentence that ends with a period: -- Keep it short (about 8-18 words). - Naturally include the exact environment name from your \`workspaceValue\` choice. - When \`requestedModelId\` is a real model id (not \`__no_model__\`) with high confidence (at least 0.9), naturally include that model's **display name** from the Available Models list. When the choice is \`__no_model__\`, or you are not highly confident the user named a model, do not mention any model. - Be dynamic and varied: do not always say "Getting started on your task in…". - Prefer lively, progressive phrasing such as "Diving into…", "Looking into…", "Checking…", "Spinning up on…". - Good examples: - - "Looking into daily environment snapshots for faster startup in App" - - "Checking mobile login redirects in Payments with Opus 4.8" - - "Digging into the flaky checkout email race in Full Stack" + - "Looking into daily environment snapshots for faster startup in App." + - "Checking mobile login redirects in Payments with Opus 4.8." + - "Digging into the flaky checkout email race in Full Stack." - The environment and model names in the sentence must match the Available lists exactly (same spelling/casing as shown). - Do not include emojis, markdown, quotes, @-mentions, or Slack markup. - Do not invent environment or model names that are not in the provided lists. diff --git a/packages/cloud-agents/src/server/router/routing-resolution.ts b/packages/cloud-agents/src/server/router/routing-resolution.ts index ebf32d35..112a837a 100644 --- a/packages/cloud-agents/src/server/router/routing-resolution.ts +++ b/packages/cloud-agents/src/server/router/routing-resolution.ts @@ -47,7 +47,7 @@ export const workspaceResponseSchema = z.object({ .string() .nullable() .describe( - 'Full short user-facing kickoff sentence posted in chat (about 8-18 words). Naturally include the exact chosen environment name, and when requestedModelId is a real model also naturally include that model display name from the Available Models list. Vary the wording; do not always use "Getting started on your task in…". No emojis, markdown, quotes, or mentions. Always provide a non-empty value for real routed tasks.', + 'Short user-facing kickoff sentence posted in chat (about 8-18 words) that ends with a period. Naturally include the exact chosen environment name, and when requestedModelId is a real model also naturally include that model display name from the Available Models list. Vary the wording; do not always use "Getting started on your task in…". No emojis, markdown, quotes, or mentions. Always provide a non-empty value for real routed tasks.', ) .optional() .default(null),