Skip to content

Commit 27a2d32

Browse files
esokulluesokullu
authored
Merge pull request #2918 from esokullu/main
Add Muse Glimmer benchmarks and improve structured completion recovery
2 parents a06c417 + bff0cbb commit 27a2d32

320 files changed

Lines changed: 193205 additions & 4 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎src/chrome/src/agent/agent.js‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import { detectProgressAction, formatLedgerRow, formatLedgerSummary, isBlockedLe
3030
import { buildGithubStargazerProgressItems } from './observers/github-stargazers.js';
3131
import { analyzeMastodonPage, mastodonHandoffInstruction, mastodonProgressGuard } from './observers/mastodon.js';
3232
import { isProgressActionAllowed, isProgressIntentActive, normalizeProgressAction, normalizeProgressIntent } from './progress-intent.js';
33-
import { classifyCompletionForm, completionDoneBlock, completionPlainFinalBlock, consumeCompletionObservation, consumeCompletionObservationResult, createCompletionInvariantState, hasUnconsumedCompletionObservation, hasUnconsumedCompletionObservationResult, recordCompletionToolResult } from './completion-invariant.js';
33+
import { classifyCompletionForm, completionDoneBlock, completionPlainFinalBlock, completionPlainFinalPartial, consumeCompletionObservation, consumeCompletionObservationResult, createCompletionInvariantState, hasUnconsumedCompletionObservation, hasUnconsumedCompletionObservationResult, recordCompletionToolResult } from './completion-invariant.js';
3434
import { cdpClient } from '../cdp/cdp-client.js';
3535
import { getActiveAdapter, getCarouselNavigationPolicy, getCarouselNavigationTarget, getFullPageCapturePolicy, getMessageRecipientGuardPolicy, parseCarouselSlideCount, UNIVERSAL_PREAMBLE } from './adapters.js';
3636
import { messageTargetMatchesObservedIdentities, normalizeMessageTarget, normalizeRecipientIdentity } from './message-recipient-guard.js';
@@ -1474,6 +1474,25 @@ export class Agent extends LoopDetector {
14741474
return completionPlainFinalBlock(this.completionInvariants.get(tabId));
14751475
}
14761476

1477+
_completionPlainFinalPartial(tabId, content, { progressBlocked = false, readBlocked = false } = {}) {
1478+
const preserveModelOutput = !progressBlocked && !readBlocked;
1479+
let partial = completionPlainFinalPartial(
1480+
this.completionInvariants.get(tabId),
1481+
preserveModelOutput ? repairAssistantDisplayText(content) : '',
1482+
{
1483+
verificationPending: this._richTextToolbarGuard.hasPending(tabId),
1484+
},
1485+
);
1486+
if (readBlocked) {
1487+
partial += '\n\nThe requested complete-thread read is still incomplete, so no whole-thread answer or summary was verified.';
1488+
}
1489+
if (progressBlocked) {
1490+
partial += '\n\nThe repeated-item task still has unresolved progress rows.';
1491+
partial = this._appendProgressLedgerToFinal(tabId, partial);
1492+
}
1493+
return partial;
1494+
}
1495+
14771496
_consumeCompletionObservation(tabId) {
14781497
const state = this.completionInvariants.get(tabId);
14791498
if (!state) return false;
@@ -27437,6 +27456,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
2743727456
let emptyOutputRecoveryAttempted = false;
2743827457
let compressionPlaceholderRecoveryAttempted = false;
2743927458
let structuredOutputRecoveryAttempted = false;
27459+
let completionPlainFinalRecoveryAttempted = false;
2744027460
let standaloneWikipediaModelSearchAttempted = false;
2744127461
let standaloneIncompleteAnswerRecoveryAttempted = false;
2744227462
let standaloneWebgpuBudgetRecoveryAttempted = false;
@@ -28083,6 +28103,20 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
2808328103
const readFinalBlock = this._readCompletenessBlock(tabId);
2808428104
const plainFinalBlocks = [progressFinalBlock, completionFinalBlock, readFinalBlock].filter(Boolean);
2808528105
if (plainFinalBlocks.length) {
28106+
if (completionFinalBlock && completionPlainFinalRecoveryAttempted) {
28107+
finalResponse = this._completionPlainFinalPartial(tabId, result.content, {
28108+
progressBlocked: !!progressFinalBlock,
28109+
readBlocked: !!readFinalBlock,
28110+
});
28111+
_traceStatus = 'partial';
28112+
messages.push({ role: 'assistant', content: finalResponse });
28113+
onUpdate('text', { content: finalResponse, replace: true });
28114+
onUpdate('warning', { message: 'Run stopped after a repeated unstructured completion response.' });
28115+
onUpdate('run_status', { status: 'partial', message: finalResponse });
28116+
await this._persistNow(tabId);
28117+
return finalResponse;
28118+
}
28119+
if (completionFinalBlock) completionPlainFinalRecoveryAttempted = true;
2808628120
messages.push(this._withResponseItems({ role: 'assistant', content: result.content }, result.responseItems, result.reasoningContent, provider));
2808728121
messages.push({ role: 'user', content: plainFinalBlocks.join('\n\n') });
2808828122
onUpdate('warning', { message: readFinalBlock
@@ -28515,6 +28549,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
2851528549
// See processMessage — used to break the empty-response→nudge cycle.
2851628550
let emptyOutputRecoveryAttempted = false;
2851728551
let compressionPlaceholderRecoveryAttempted = false;
28552+
let completionPlainFinalRecoveryAttempted = false;
2851828553
let standaloneWikipediaModelSearchAttempted = false;
2851928554
let standaloneIncompleteAnswerRecoveryAttempted = false;
2852028555
let standaloneWebgpuBudgetRecoveryAttempted = false;
@@ -28925,6 +28960,19 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
2892528960
const readFinalBlock = this._readCompletenessBlock(tabId);
2892628961
const plainFinalBlocks = [progressFinalBlock, completionFinalBlock, readFinalBlock].filter(Boolean);
2892728962
if (plainFinalBlocks.length) {
28963+
if (completionFinalBlock && completionPlainFinalRecoveryAttempted) {
28964+
const partial = this._completionPlainFinalPartial(tabId, fullText, {
28965+
progressBlocked: !!progressFinalBlock,
28966+
readBlocked: !!readFinalBlock,
28967+
});
28968+
messages.push({ role: 'assistant', content: partial });
28969+
onUpdate('text', { content: partial, replace: true });
28970+
onUpdate('warning', { message: 'Run stopped after a repeated unstructured completion response.' });
28971+
onUpdate('run_status', { status: 'partial', message: partial });
28972+
await this._persistNow(tabId);
28973+
return finish(partial, 'partial');
28974+
}
28975+
if (completionFinalBlock) completionPlainFinalRecoveryAttempted = true;
2892828976
messages.push(this._withResponseItems({ role: 'assistant', content: fullText }, responseItems, reasoningContent, provider));
2892928977
messages.push({ role: 'user', content: plainFinalBlocks.join('\n\n') });
2893028978
if (completionFinalBlock || readFinalBlock) onUpdate('text', { content: '', replace: true });

‎src/chrome/src/agent/completion-invariant.js‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -523,3 +523,24 @@ export function completionPlainFinalBlock(state) {
523523
if (!state?.hadAction) return null;
524524
return '[RUNTIME COMPLETION BLOCK: This Act/Dev run executed a consequential action, so a plain final answer cannot end it. Call done with an explicit outcome of success, partial, or failed. Use success only after a post-action observation verified the current state.]';
525525
}
526+
527+
export function completionPlainFinalPartial(state, content, { verificationPending = false } = {}) {
528+
const mustVerify = verificationPending
529+
|| !!state?.verificationDebt
530+
|| !!state?.iframeFormVerificationDebt;
531+
const lines = [
532+
'The run stopped after the model returned plain text twice instead of the required structured completion.',
533+
'Outcome: partial.',
534+
];
535+
if (mustVerify) {
536+
lines.push('A consequential action may have occurred, but its final state was not verified. Inspect the current page before retrying so the action is not repeated blindly.');
537+
return lines.join('\n\n');
538+
}
539+
lines.push('The latest page state was observed, but the model did not complete the required done handoff.');
540+
const summary = String(content || '').trim();
541+
if (summary) {
542+
const bounded = summary.length > 4000 ? `${summary.slice(0, 4000)}\n[Latest model output truncated]` : summary;
543+
lines.push(`Latest model output (not accepted as verified completion):\n${bounded}`);
544+
}
545+
return lines.join('\n\n');
546+
}

‎src/chrome/src/agent/scheduler.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1727,6 +1727,7 @@ export class ScheduledJobManager {
17271727
// Persist an explicit verdict for Ask runs (they never emit a done
17281728
// update): downstream badge styling must not guess from null.
17291729
const effectiveOutcome = runOutcome
1730+
?? normalizeDoneOutcome(runStatus)
17301731
?? ((running.mode || 'act') === 'ask' && askRunSucceeded(result, sawFailureLikeUpdate)
17311732
? 'success'
17321733
: null);

‎src/firefox/src/agent/agent.js‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import { detectProgressAction, formatLedgerRow, formatLedgerSummary, isBlockedLe
3030
import { buildGithubStargazerProgressItems } from './observers/github-stargazers.js';
3131
import { analyzeMastodonPage, mastodonHandoffInstruction, mastodonProgressGuard } from './observers/mastodon.js';
3232
import { isProgressActionAllowed, isProgressIntentActive, normalizeProgressAction, normalizeProgressIntent } from './progress-intent.js';
33-
import { classifyCompletionForm, completionDoneBlock, completionPlainFinalBlock, consumeCompletionObservation, consumeCompletionObservationResult, createCompletionInvariantState, hasUnconsumedCompletionObservation, hasUnconsumedCompletionObservationResult, recordCompletionToolResult } from './completion-invariant.js';
33+
import { classifyCompletionForm, completionDoneBlock, completionPlainFinalBlock, completionPlainFinalPartial, consumeCompletionObservation, consumeCompletionObservationResult, createCompletionInvariantState, hasUnconsumedCompletionObservation, hasUnconsumedCompletionObservationResult, recordCompletionToolResult } from './completion-invariant.js';
3434
import { getActiveAdapter, getCarouselNavigationPolicy, getCarouselNavigationTarget, getMessageRecipientGuardPolicy, parseCarouselSlideCount, UNIVERSAL_PREAMBLE } from './adapters.js';
3535
import { messageTargetMatchesObservedIdentities, normalizeMessageTarget, normalizeRecipientIdentity } from './message-recipient-guard.js';
3636
import {
@@ -1190,6 +1190,25 @@ export class Agent extends LoopDetector {
11901190
return completionPlainFinalBlock(this.completionInvariants.get(tabId));
11911191
}
11921192

1193+
_completionPlainFinalPartial(tabId, content, { progressBlocked = false, readBlocked = false } = {}) {
1194+
const preserveModelOutput = !progressBlocked && !readBlocked;
1195+
let partial = completionPlainFinalPartial(
1196+
this.completionInvariants.get(tabId),
1197+
preserveModelOutput ? repairAssistantDisplayText(content) : '',
1198+
{
1199+
verificationPending: this._richTextToolbarGuard.hasPending(tabId),
1200+
},
1201+
);
1202+
if (readBlocked) {
1203+
partial += '\n\nThe requested complete-thread read is still incomplete, so no whole-thread answer or summary was verified.';
1204+
}
1205+
if (progressBlocked) {
1206+
partial += '\n\nThe repeated-item task still has unresolved progress rows.';
1207+
partial = this._appendProgressLedgerToFinal(tabId, partial);
1208+
}
1209+
return partial;
1210+
}
1211+
11931212
_consumeCompletionObservation(tabId) {
11941213
const state = this.completionInvariants.get(tabId);
11951214
if (!state) return false;
@@ -21066,6 +21085,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
2106621085
let emptyOutputRecoveryAttempted = false;
2106721086
let compressionPlaceholderRecoveryAttempted = false;
2106821087
let structuredOutputRecoveryAttempted = false;
21088+
let completionPlainFinalRecoveryAttempted = false;
2106921089
let askStreamingDisabledForRun = false;
2107021090

2107121091
// Keep trace persistence ordered without putting IndexedDB on the token
@@ -21622,6 +21642,20 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
2162221642
const readFinalBlock = this._readCompletenessBlock(tabId);
2162321643
const plainFinalBlocks = [progressFinalBlock, completionFinalBlock, readFinalBlock].filter(Boolean);
2162421644
if (plainFinalBlocks.length) {
21645+
if (completionFinalBlock && completionPlainFinalRecoveryAttempted) {
21646+
finalResponse = this._completionPlainFinalPartial(tabId, result.content, {
21647+
progressBlocked: !!progressFinalBlock,
21648+
readBlocked: !!readFinalBlock,
21649+
});
21650+
_traceStatus = 'partial';
21651+
messages.push({ role: 'assistant', content: finalResponse });
21652+
onUpdate('text', { content: finalResponse, replace: true });
21653+
onUpdate('warning', { message: 'Run stopped after a repeated unstructured completion response.' });
21654+
onUpdate('run_status', { status: 'partial', message: finalResponse });
21655+
await this._persistNow(tabId);
21656+
return finalResponse;
21657+
}
21658+
if (completionFinalBlock) completionPlainFinalRecoveryAttempted = true;
2162521659
messages.push(this._withResponseItems({ role: 'assistant', content: result.content }, result.responseItems, result.reasoningContent, provider));
2162621660
messages.push({ role: 'user', content: plainFinalBlocks.join('\n\n') });
2162721661
onUpdate('warning', { message: readFinalBlock
@@ -21972,6 +22006,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
2197222006
// See processMessage — used to break the empty-response→nudge cycle.
2197322007
let emptyOutputRecoveryAttempted = false;
2197422008
let compressionPlaceholderRecoveryAttempted = false;
22009+
let completionPlainFinalRecoveryAttempted = false;
2197522010
let pendingVisionFallbackMessages = null;
2197622011
let visionFallbackAttempted = false;
2197722012
let streamEmittedOutput = false;
@@ -22328,6 +22363,19 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
2232822363
const readFinalBlock = this._readCompletenessBlock(tabId);
2232922364
const plainFinalBlocks = [progressFinalBlock, completionFinalBlock, readFinalBlock].filter(Boolean);
2233022365
if (plainFinalBlocks.length) {
22366+
if (completionFinalBlock && completionPlainFinalRecoveryAttempted) {
22367+
const partial = this._completionPlainFinalPartial(tabId, fullText, {
22368+
progressBlocked: !!progressFinalBlock,
22369+
readBlocked: !!readFinalBlock,
22370+
});
22371+
messages.push({ role: 'assistant', content: partial });
22372+
onUpdate('text', { content: partial, replace: true });
22373+
onUpdate('warning', { message: 'Run stopped after a repeated unstructured completion response.' });
22374+
onUpdate('run_status', { status: 'partial', message: partial });
22375+
await this._persistNow(tabId);
22376+
return finish(partial, 'partial');
22377+
}
22378+
if (completionFinalBlock) completionPlainFinalRecoveryAttempted = true;
2233122379
messages.push(this._withResponseItems({ role: 'assistant', content: fullText }, responseItems, reasoningContent, provider));
2233222380
messages.push({ role: 'user', content: plainFinalBlocks.join('\n\n') });
2233322381
if (completionFinalBlock || readFinalBlock) onUpdate('text', { content: '', replace: true });

‎src/firefox/src/agent/completion-invariant.js‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -523,3 +523,24 @@ export function completionPlainFinalBlock(state) {
523523
if (!state?.hadAction) return null;
524524
return '[RUNTIME COMPLETION BLOCK: This Act/Dev run executed a consequential action, so a plain final answer cannot end it. Call done with an explicit outcome of success, partial, or failed. Use success only after a post-action observation verified the current state.]';
525525
}
526+
527+
export function completionPlainFinalPartial(state, content, { verificationPending = false } = {}) {
528+
const mustVerify = verificationPending
529+
|| !!state?.verificationDebt
530+
|| !!state?.iframeFormVerificationDebt;
531+
const lines = [
532+
'The run stopped after the model returned plain text twice instead of the required structured completion.',
533+
'Outcome: partial.',
534+
];
535+
if (mustVerify) {
536+
lines.push('A consequential action may have occurred, but its final state was not verified. Inspect the current page before retrying so the action is not repeated blindly.');
537+
return lines.join('\n\n');
538+
}
539+
lines.push('The latest page state was observed, but the model did not complete the required done handoff.');
540+
const summary = String(content || '').trim();
541+
if (summary) {
542+
const bounded = summary.length > 4000 ? `${summary.slice(0, 4000)}\n[Latest model output truncated]` : summary;
543+
lines.push(`Latest model output (not accepted as verified completion):\n${bounded}`);
544+
}
545+
return lines.join('\n\n');
546+
}

‎src/firefox/src/agent/scheduler.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1703,6 +1703,7 @@ export class ScheduledJobManager {
17031703
// Persist an explicit verdict for Ask runs (they never emit a done
17041704
// update): downstream badge styling must not guess from null.
17051705
const effectiveOutcome = runOutcome
1706+
?? normalizeDoneOutcome(runStatus)
17061707
?? ((running.mode || 'act') === 'ask' && askRunSucceeded(result, sawFailureLikeUpdate)
17071708
? 'success'
17081709
: null);

‎test/llm/results-scenarios/openrouter-muse-glimmer-30b-retry-scenarios-20260825_chrome_meta_muse-glimmer-30b/002.json‎

Lines changed: 103 additions & 0 deletions
Large diffs are not rendered by default.

‎test/llm/results-scenarios/openrouter-muse-glimmer-30b-retry-scenarios-20260825_chrome_meta_muse-glimmer-30b/017.json‎

Lines changed: 99 additions & 0 deletions
Large diffs are not rendered by default.

‎test/llm/results-scenarios/openrouter-muse-glimmer-30b-retry-scenarios-20260825_chrome_meta_muse-glimmer-30b/021.json‎

Lines changed: 94 additions & 0 deletions
Large diffs are not rendered by default.

‎test/llm/results-scenarios/openrouter-muse-glimmer-30b-retry-scenarios-20260825_chrome_meta_muse-glimmer-30b/086.json‎

Lines changed: 90 additions & 0 deletions
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)