Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion tools/loop-metrics/dist/metrics.js
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,25 @@ export function aggregateMetrics(entries) {
let totalDurationS = 0;
let totalActionsTaken = 0;
let totalEscalations = 0;
let runsWithoutEscalation = 0;
for (const entry of entries) {
totalTokens += entry.tokens_estimate || 0;
totalDurationS += entry.duration_s || 0;
totalActionsTaken += entry.actions_taken || 0;
totalEscalations += entry.escalations || 0;
if (!entry.escalations)
runsWithoutEscalation++;
}
const totalRuns = entries.length;
const successRatePct = totalRuns > 0 ? ((totalRuns - totalEscalations) / totalRuns) * 100 : 0;
// totalEscalations is a sum of each run's escalation *count* (a run can
// log more than one, e.g. several items escalated in one triage pass) --
// subtracting it from totalRuns (a count of runs) conflates events with
// runs and can go negative for a real, ordinary run log (this repo's own
// loop-run-log.md has entries with escalations: 4 and escalations: 5).
// Success rate is "what fraction of runs didn't escalate at all", which
// stays correctly bounded to [0, 100] regardless of how many escalations
// any single run logged.
const successRatePct = totalRuns > 0 ? (runsWithoutEscalation / totalRuns) * 100 : 0;
const avgDurationS = totalRuns > 0 ? totalDurationS / totalRuns : 0;
// Simple heuristic: Each successful action is worth +10, each escalation is -5.
const roiScore = (totalActionsTaken * 10) - (totalEscalations * 5);
Expand Down
12 changes: 11 additions & 1 deletion tools/loop-metrics/src/metrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,16 +77,26 @@ export function aggregateMetrics(entries: RunEntry[]): MetricsDashboard {
let totalDurationS = 0;
let totalActionsTaken = 0;
let totalEscalations = 0;
let runsWithoutEscalation = 0;

for (const entry of entries) {
totalTokens += entry.tokens_estimate || 0;
totalDurationS += entry.duration_s || 0;
totalActionsTaken += entry.actions_taken || 0;
totalEscalations += entry.escalations || 0;
if (!entry.escalations) runsWithoutEscalation++;
}

const totalRuns = entries.length;
const successRatePct = totalRuns > 0 ? ((totalRuns - totalEscalations) / totalRuns) * 100 : 0;
// totalEscalations is a sum of each run's escalation *count* (a run can
// log more than one, e.g. several items escalated in one triage pass) --
// subtracting it from totalRuns (a count of runs) conflates events with
// runs and can go negative for a real, ordinary run log (this repo's own
// loop-run-log.md has entries with escalations: 4 and escalations: 5).
// Success rate is "what fraction of runs didn't escalate at all", which
// stays correctly bounded to [0, 100] regardless of how many escalations
// any single run logged.
const successRatePct = totalRuns > 0 ? (runsWithoutEscalation / totalRuns) * 100 : 0;
const avgDurationS = totalRuns > 0 ? totalDurationS / totalRuns : 0;

// Simple heuristic: Each successful action is worth +10, each escalation is -5.
Expand Down
18 changes: 18 additions & 0 deletions tools/loop-metrics/test/metrics.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,24 @@ test('loop-metrics filters and aggregates', () => {
assert.strictEqual(metrics.totalActionsTaken, 3);
assert.strictEqual(metrics.totalEscalations, 1);
assert.strictEqual(metrics.roiScore, (3 * 10) - (1 * 5)); // 25
assert.strictEqual(metrics.successRatePct, 50); // 1 of 2 runs had no escalation
});

test('successRatePct stays within [0, 100] when a single run logs more than one escalation', () => {
// A run can escalate several items in one pass (this repo's own
// loop-run-log.md has real entries with escalations: 4 and 5), so
// totalEscalations (a sum of per-run counts) can exceed totalRuns.
// successRatePct must still reflect "runs that didn't escalate", not go
// negative from subtracting an event count as if it were a run count.
const entries = [
{ run_id: 'a', pattern: 'ci-sweeper', duration_s: 1, items_found: 5, actions_taken: 1, escalations: 5, tokens_estimate: 1000, outcome: 'escalated' },
{ run_id: 'b', pattern: 'ci-sweeper', duration_s: 1, items_found: 0, actions_taken: 0, escalations: 0, tokens_estimate: 1000, outcome: 'report-only' },
];

const metrics = aggregateMetrics(entries);
assert.strictEqual(metrics.totalEscalations, 5);
assert.strictEqual(metrics.successRatePct, 50); // 1 of 2 runs had no escalation
assert.ok(metrics.successRatePct >= 0 && metrics.successRatePct <= 100);
});

test('filterEntries keeps entries with unparseable run_id when a timeframe is set', () => {
Expand Down