[Improve] Guide kickoff messages to end with a period

packages/cloud-agents/evals/router/assertions/routing-assertions.ts

@@ -7,6 +7,7 @@ interface RoutingResponse {
   workspaceValue?: string;
   reasoning?: string;
   confidence?: number;
+  kickoffMessage?: string | null;
   needsExternalLookup?: boolean;
   externalReference?: string | null;
   requestedModelId?: string | null;
@@ -272,6 +273,49 @@ function requestedModelIdMatchesExpected(
   };
 }
 
+/**
+ * Asserts that kickoffMessage is present as short display text ending with a
+ * single period (the router prompt contract for chat started messages).
+ */
+function hasValidKickoffMessage(output: string): AssertionResult {
+  const json = extractJson(output);
+  if (!json) {
+    return { pass: false, score: 0, reason: 'Invalid JSON response' };
+  }
+
+  const kickoff = json.kickoffMessage;
+  if (typeof kickoff !== 'string' || kickoff.trim().length === 0) {
+    return {
+      pass: false,
+      score: 0,
+      reason: 'kickoffMessage is missing or empty',
+    };
+  }
+
+  const text = kickoff.trim();
+  if (!text.endsWith('.')) {
+    return {
+      pass: false,
+      score: 0,
+      reason: `kickoffMessage must end with a period: ${text}`,
+    };
+  }
+
+  if (text.length < 8 || text.length > 200) {
+    return {
+      pass: false,
+      score: 0,
+      reason: `kickoffMessage length looks off (${text.length} chars): ${text}`,
+    };
+  }
+
+  return {
+    pass: true,
+    score: 1,
+    reason: `Valid kickoffMessage: ${text}`,
+  };
+}
+
 /**
  * Asserts that the router did not report a model preference when the user did
  * not express one: requestedModelId must be the explicit `__no_model__`
@@ -304,6 +348,7 @@ export {
   workspaceValueMatchesExpected,
   reasoningContainsExpected,
   requestedModelIdMatchesExpected,
+  hasValidKickoffMessage,
   requestedModelIdIsNull,
   extractJson,
 };

packages/cloud-agents/evals/router/datasets/basic.yaml

@@ -27,6 +27,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
 
 - description: 'Explanation task - understanding code behavior'
   vars:
@@ -54,6 +56,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
 
 - description: 'Planning task - architecture design'
   vars:
@@ -81,6 +85,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
 
 - description: 'Ambiguous repository case'
   vars:
@@ -104,6 +110,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     # Should pick a specific repository (best guess), NOT all_repositories
     - type: javascript
       value: file://assertions/routing-assertions.ts:hasValidWorkspaceValue
@@ -133,6 +141,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:hasValidWorkspaceValue
 
@@ -163,6 +173,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected
 
@@ -190,6 +202,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
 
 - description: 'Refactoring task - should route to Coder'
   vars:
@@ -219,6 +233,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
 
 - description: 'Explicit project selection - user forces specific environment'
   vars:
@@ -246,6 +262,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:hasValidWorkspaceValue
     - type: javascript
@@ -276,6 +294,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected
 
@@ -297,6 +317,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
 
 - description: 'Investigative question about UI regression - should route to Explainer with cloud app environment'
   vars:
@@ -326,6 +348,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:hasValidWorkspaceValue
     - type: javascript
@@ -359,6 +383,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:hasValidWorkspaceValue
     - type: javascript
@@ -390,6 +416,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:hasValidWorkspaceValue
     - type: javascript
@@ -422,6 +450,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:hasValidWorkspaceValue
     - type: javascript
@@ -453,6 +483,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:hasValidWorkspaceValue
     - type: javascript

packages/cloud-agents/evals/router/datasets/model-routing.yaml

@@ -26,6 +26,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected
     - type: javascript
@@ -54,6 +56,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected
     - type: javascript
@@ -82,6 +86,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected
     - type: javascript
@@ -110,6 +116,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected
     - type: javascript
@@ -138,6 +146,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected
     - type: javascript
@@ -166,6 +176,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected
     - type: javascript
@@ -194,6 +206,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected
     - type: javascript
@@ -222,6 +236,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected
     - type: javascript
@@ -249,6 +265,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected
     - type: javascript
@@ -275,6 +293,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected
     - type: javascript
@@ -305,6 +325,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected
     - type: javascript
@@ -334,6 +356,8 @@
   assert:
     - type: javascript
       value: file://assertions/routing-assertions.ts:isValidRoutingJson
+    - type: javascript
+      value: file://assertions/routing-assertions.ts:hasValidKickoffMessage
     - type: javascript
       value: file://assertions/routing-assertions.ts:workspaceValueMatchesExpected
     - type: javascript

packages/cloud-agents/evals/router/prompts/routing.ts

@@ -149,7 +149,7 @@ Respond with a single JSON object containing exactly these fields:
 - "workspaceValue" (string): Name of the chosen environment.
 - "reasoning" (string): Brief explanation of your workspace decision.
 - "confidence" (number): Confidence in the workspace choice from 0 to 1.
-- "kickoffMessage" (string or null): Full short user-facing kickoff sentence (about 8-18 words). Naturally include the chosen environment name, and naturally include the model display name when requestedModelId is a real model. Vary wording; no "Getting started on your task in…" boilerplate every time. No emojis, markdown, quotes, or mentions.
+- "kickoffMessage" (string): Required short user-facing kickoff sentence (about 8-18 words) that ends with a period. Naturally include the chosen environment name, and naturally include the model display name when requestedModelId is a real model. Vary wording; no "Getting started on your task in…" boilerplate every time. No emojis, markdown, quotes, or mentions. Always provide a non-empty value for real routed tasks.
 - "needsExternalLookup" (boolean): Whether an external reference must be fetched before routing, per the external lookup rules.
 - "externalReference" (string or null): The exact external reference to fetch when needsExternalLookup is true, otherwise null.
 - "requestedModelId" (string or null): The model id the user explicitly requested from the Available Models list, or the literal "${NO_MODEL_MENTIONED_VALUE}" when the user does not name a model.
@@ -160,7 +160,9 @@ Respond with a single JSON object containing exactly these fields:
 export default function generatePrompt({ vars }: PromptInput): string {
   const context = buildContext(vars);
   const availableModels = buildAvailableModelsSection(vars.extraModels);
-  const routingPrompt = buildWorkspaceRoutingPrompt();
+  const routingPrompt = buildWorkspaceRoutingPrompt({
+    includeKickoffMessage: true,
+  });
 
   return `${routingPrompt}
 

packages/cloud-agents/src/server/router/prompts/routing-prompt.ts

@@ -42,17 +42,16 @@ const EXTERNAL_LOOKUP_RULES = `**External lookup rules:**
 
 const KICKOFF_MESSAGE_SECTION = `## Kickoff Message
 
-Also always set \`kickoffMessage\` to the full short user-facing kickoff for chat (one brief sentence). Roomote posts this text as-is, so weave the details in naturally instead of using a fixed template:
+Also always set \`kickoffMessage\` to a short user-facing kickoff sentence for chat (about 8-18 words). Roomote posts this text as-is, so weave the details in naturally instead of using a fixed template. Write a complete sentence that ends with a period:
 
-- Keep it short (about 8-18 words).
 - Naturally include the exact environment name from your \`workspaceValue\` choice.
 - When \`requestedModelId\` is a real model id (not \`__no_model__\`) with high confidence (at least 0.9), naturally include that model's **display name** from the Available Models list. When the choice is \`__no_model__\`, or you are not highly confident the user named a model, do not mention any model.
 - Be dynamic and varied: do not always say "Getting started on your task in…".
 - Prefer lively, progressive phrasing such as "Diving into…", "Looking into…", "Checking…", "Spinning up on…".
 - Good examples:
-  - "Looking into daily environment snapshots for faster startup in App"
-  - "Checking mobile login redirects in Payments with Opus 4.8"
-  - "Digging into the flaky checkout email race in Full Stack"
+  - "Looking into daily environment snapshots for faster startup in App."
+  - "Checking mobile login redirects in Payments with Opus 4.8."
+  - "Digging into the flaky checkout email race in Full Stack."
 - The environment and model names in the sentence must match the Available lists exactly (same spelling/casing as shown).
 - Do not include emojis, markdown, quotes, @-mentions, or Slack markup.
 - Do not invent environment or model names that are not in the provided lists.

packages/cloud-agents/src/server/router/routing-resolution.ts

@@ -47,7 +47,7 @@ export const workspaceResponseSchema = z.object({
     .string()
     .nullable()
     .describe(
-      'Full short user-facing kickoff sentence posted in chat (about 8-18 words). Naturally include the exact chosen environment name, and when requestedModelId is a real model also naturally include that model display name from the Available Models list. Vary the wording; do not always use "Getting started on your task in…". No emojis, markdown, quotes, or mentions. Always provide a non-empty value for real routed tasks.',
+      'Short user-facing kickoff sentence posted in chat (about 8-18 words) that ends with a period. Naturally include the exact chosen environment name, and when requestedModelId is a real model also naturally include that model display name from the Available Models list. Vary the wording; do not always use "Getting started on your task in…". No emojis, markdown, quotes, or mentions. Always provide a non-empty value for real routed tasks.',
     )
     .optional()
     .default(null),