Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 84 additions & 15 deletions internal/engine/engine_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1688,28 +1688,33 @@ func TestReportOverviewMarkdownIncludesCISummary(t *testing.T) {
Name: "good|session",
Health: 92,
Metrics: Metrics{
SourceTool: "aider",
ModelUsed: "gpt-4.1",
TokensInput: 1000,
TokensOutput: 500,
TokensCacheW: 25,
TokensCacheR: 50,
ToolCallsOK: 4,
ToolCallsFail: 1,
CostEstimated: 0.12,
SourceTool: "aider",
ModelUsed: "gpt-4.1",
TokensInput: 1000,
TokensOutput: 500,
TokensCacheW: 25,
TokensCacheR: 50,
ToolCallsOK: 4,
ToolCallsFail: 1,
ToolUsage: map[string]int{"read_file": 5},
AssistantTurns: 2,
DurationSec: 45,
CostEstimated: 0.12,
},
},
{
Name: "bad",
Health: 30,
Anomalies: []Anomaly{{Type: "hanging", Severity: SeverityHigh}},
Metrics: Metrics{
SourceTool: "cursor",
ModelUsed: "default",
TokensInput: 300,
TokensOutput: 200,
ToolCallsFail: 5,
CostEstimated: 0.34,
SourceTool: "cursor",
ModelUsed: "default",
TokensInput: 300,
TokensOutput: 200,
ToolCallsFail: 5,
AssistantTurns: 1,
DurationSec: 90,
CostEstimated: 0.34,
},
},
}
Expand All @@ -1720,6 +1725,9 @@ func TestReportOverviewMarkdownIncludesCISummary(t *testing.T) {
"| Total tokens | 2075 |",
"| Health Trend |",
"| Tool failures | 6 / 10 (60.0%) |",
"## Incident timeline",
"Last milestone",
"Touched surface",
"| Aider | 1 |",
"| Cursor | 1 |",
"good\\|session",
Expand All @@ -1731,6 +1739,67 @@ func TestReportOverviewMarkdownIncludesCISummary(t *testing.T) {
}
}

func TestBuildIncidentTimelineKeepsEvidenceCompact(t *testing.T) {
got := BuildIncidentTimeline(Session{
Name: "incident",
Metrics: Metrics{
AssistantTurns: 4,
DurationSec: 125,
GapsSec: []float64{2, 75},
ToolCallsOK: 3,
ToolCallsFail: 2,
ToolUsage: map[string]int{"read_file": 2, "shell\nsecret args": 3},
TokensInput: 42000,
TokensOutput: 9000,
CostEstimated: 0.80,
},
LoopFingerprints: []LoopFingerprint{{ToolName: "shell\nsecret args", Count: 3, Severity: "critical"}},
LoopCost: LoopCost{TotalLoopCost: 0.30},
})
if len(got.Items) < 5 {
t.Fatalf("expected compact incident evidence, got %+v", got.Items)
}
kinds := map[string]bool{}
for _, item := range got.Items {
kinds[item.Kind] = true
if strings.Contains(item.Detail, "\n") {
t.Fatalf("timeline detail should stay one-line and redacted enough for reports: %+v", item)
}
if strings.Contains(item.Detail, "secret") {
t.Fatalf("timeline detail should not expose whitespace-suffixed tool text: %+v", item)
}
}
for _, want := range []string{"milestone", "idle_gap", "failure_loop", "touched_surface", "burn_divergence"} {
if !kinds[want] {
t.Fatalf("missing incident timeline kind %q in %+v", want, got.Items)
}
}
}

func TestReportOverviewJSONIncludesIncidentTimelines(t *testing.T) {
sessions := []Session{{
Name: "slow",
Health: 45,
Metrics: Metrics{
AssistantTurns: 3,
DurationSec: 180,
GapsSec: []float64{90},
ToolCallsOK: 1,
ToolCallsFail: 2,
ToolUsage: map[string]int{"shell": 3},
},
}}
var payload struct {
IncidentTimelines []IncidentTimelineSummary `json:"incident_timelines"`
}
if err := json.Unmarshal([]byte(ReportOverviewJSON(ComputeOverview(sessions), sessions)), &payload); err != nil {
t.Fatalf("parse overview json: %v", err)
}
if len(payload.IncidentTimelines) != 1 || len(payload.IncidentTimelines[0].Items) == 0 {
t.Fatalf("expected incident timeline JSON evidence, got %+v", payload.IncidentTimelines)
}
}

func TestReportOverviewCostLabelUsesEstimatedTotalCost(t *testing.T) {
sessions := []Session{{
Name: "demo",
Expand Down
162 changes: 162 additions & 0 deletions internal/engine/incident_timeline.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
package engine

import (
"fmt"
"math"
"sort"
"strings"

"github.com/luoyuctl/agenttrace/internal/i18n"
)

type IncidentTimelineItem struct {
Kind string `json:"kind"`
Label string `json:"label"`
Detail string `json:"detail"`
Severity string `json:"severity"`
}

type IncidentTimelineSummary struct {
Session string `json:"session"`
Items []IncidentTimelineItem `json:"items"`
}

func BuildIncidentTimeline(s Session) IncidentTimelineSummary {
m := s.Metrics
summary := IncidentTimelineSummary{Session: s.Name}
add := func(kind, label, detail, severity string) {
detail = strings.TrimSpace(detail)
if detail == "" {
return
}
summary.Items = append(summary.Items, IncidentTimelineItem{
Kind: kind, Label: label, Detail: detail, Severity: severity,
})
}

if m.AssistantTurns > 0 {
detail := fmt.Sprintf(i18n.T("incident_milestone_detail"), m.AssistantTurns, FmtDuration(m.DurationSec))
add("milestone", i18n.T("incident_milestone"), detail, SeverityLow)
}

if gap := maxIncidentGap(m.GapsSec); gap >= 30 {
severity := SeverityLow
if gap >= 300 {
severity = SeverityHigh
} else if gap >= 60 {
severity = SeverityMedium
}
add("idle_gap", i18n.T("incident_idle_gap"), fmt.Sprintf(i18n.T("incident_idle_gap_detail"), gap), severity)
}

if fp, ok := topIncidentLoop(s.LoopFingerprints); ok {
tool := incidentSafeName(fp.ToolName)
if tool == "" {
tool = i18n.T("unknown_tool")
}
add("failure_loop", i18n.T("incident_failure_loop"), fmt.Sprintf(i18n.T("incident_failure_loop_detail"), tool, fp.Count), incidentLoopSeverity(fp.Severity))
} else if totalTools := m.ToolCallsOK + m.ToolCallsFail; totalTools > 0 && m.ToolCallsFail > 1 {
failRate := float64(m.ToolCallsFail) / float64(totalTools) * 100
severity := SeverityMedium
if failRate >= 30 {
severity = SeverityHigh
}
add("failure_loop", i18n.T("incident_failure_loop"), fmt.Sprintf(i18n.T("incident_failure_rate_detail"), m.ToolCallsFail, totalTools, failRate), severity)
}

if totalTools := m.ToolCallsOK + m.ToolCallsFail; totalTools > 0 && len(m.ToolUsage) > 0 {
name, count := topIncidentTool(m.ToolUsage)
if count > 0 {
add("touched_surface", i18n.T("incident_touched_surface"), fmt.Sprintf(i18n.T("incident_touched_surface_detail"), len(m.ToolUsage), totalTools, incidentSafeName(name), count), SeverityLow)
}
}

totalTokens := m.TokensInput + m.TokensOutput + m.TokensCacheW + m.TokensCacheR
if s.LoopCost.TotalLoopCost > 0 && m.CostEstimated > 0 {
loopCost := ClampLoopWaste(s.LoopCost.TotalLoopCost, m.CostEstimated)
add("burn_divergence", i18n.T("incident_burn_divergence"), fmt.Sprintf(i18n.T("incident_burn_loop_detail"), loopCost, m.CostEstimated), SeverityHigh)
} else if totalTokens > 0 && m.AssistantTurns > 0 {
tokensPerTurn := totalTokens / m.AssistantTurns
if tokensPerTurn >= 10000 {
add("burn_divergence", i18n.T("incident_burn_divergence"), fmt.Sprintf(i18n.T("incident_burn_tokens_detail"), tokensPerTurn, m.AssistantTurns), SeverityMedium)
}
}

return summary
}

func maxIncidentGap(gaps []float64) float64 {
maxGap := 0.0
for _, gap := range gaps {
if math.IsNaN(gap) || math.IsInf(gap, 0) || gap < 0 {
continue
}
if gap > maxGap {
maxGap = gap
}
}
return maxGap
}

func topIncidentLoop(items []LoopFingerprint) (LoopFingerprint, bool) {
if len(items) == 0 {
return LoopFingerprint{}, false
}
ordered := append([]LoopFingerprint(nil), items...)
sort.SliceStable(ordered, func(i, j int) bool {
if ordered[i].Count == ordered[j].Count {
if ordered[i].ToolName == ordered[j].ToolName {
return ordered[i].ResultHash < ordered[j].ResultHash
}
return ordered[i].ToolName < ordered[j].ToolName
}
return ordered[i].Count > ordered[j].Count
})
return ordered[0], true
}

func topIncidentTool(items map[string]int) (string, int) {
type kv struct {
name string
count int
}
ordered := make([]kv, 0, len(items))
for name, count := range items {
if count <= 0 {
continue
}
ordered = append(ordered, kv{name: name, count: count})
}
if len(ordered) == 0 {
return "", 0
}
sort.SliceStable(ordered, func(i, j int) bool {
if ordered[i].count == ordered[j].count {
return ordered[i].name < ordered[j].name
}
return ordered[i].count > ordered[j].count
})
return ordered[0].name, ordered[0].count
}

func incidentLoopSeverity(severity string) string {
switch strings.ToLower(severity) {
case SeverityHigh, "critical":
return SeverityHigh
case SeverityMedium, "warning":
return SeverityMedium
default:
return SeverityLow
}
}

func incidentSafeName(value string) string {
value = strings.TrimSpace(value)
if fields := strings.Fields(value); len(fields) > 0 {
value = fields[0]
}
if len(value) > 48 {
value = value[:48]
}
return value
}
70 changes: 66 additions & 4 deletions internal/engine/report.go
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,7 @@ func ReportOverviewJSON(ov Overview, sessions []Session) string {
for _, p := range trend.Points {
points = append(points, trendPoint{Name: p.Name, Health: p.Health, Cost: round4(p.Cost)})
}
incidentTimelines := overviewIncidentTimelines(orderedSessions, 10)

payload := map[string]interface{}{
"version": Version,
Expand Down Expand Up @@ -495,10 +496,11 @@ func ReportOverviewJSON(ov Overview, sessions []Session) string {
"files": sortedReportKeys(fileSurface),
"high_authority_tools": highAuthorityTools(sortedReportKeys(toolSurface)),
},
"by_agent": agents,
"by_model": models,
"recent_sessions": recent,
"anomalies": anomaliesReturned,
"by_agent": agents,
"by_model": models,
"recent_sessions": recent,
"incident_timelines": incidentTimelines,
"anomalies": anomaliesReturned,
}
out, _ := json.MarshalIndent(payload, "", " ")
return string(out)
Expand All @@ -521,6 +523,25 @@ func ReportOverviewMarkdown(ov Overview, sessions []Session) string {
fmt.Fprintf(&b, "| %s | %d |\n", i18n.T("report_total_tokens"), summary.TotalTokens)
fmt.Fprintf(&b, "| %s | %d / %d (%.1f%%) |\n\n", i18n.T("report_tool_failures"), summary.FailedTools, summary.TotalTools, summary.ToolFailRate)

fmt.Fprintf(&b, "## %s\n\n", i18n.T("incident_timeline_title"))
timelines := overviewIncidentTimelines(orderedSessions, 6)
if len(timelines) == 0 {
fmt.Fprintf(&b, "%s\n\n", i18n.T("incident_timeline_no_evidence"))
} else {
fmt.Fprintf(&b, "| %s | %s | %s | %s |\n|---|---|---|---|\n",
i18n.T("report_session"), i18n.T("incident_timeline_signal"), i18n.T("incident_timeline_evidence"), i18n.T("incident_timeline_severity"))
for _, timeline := range timelines {
for _, item := range timeline.Items {
fmt.Fprintf(&b, "| %s | %s | %s | %s |\n",
markdownCell(timeline.Session),
markdownCell(item.Label),
markdownCell(item.Detail),
markdownCell(reportSeverityLabel(item.Severity)))
}
}
fmt.Fprintln(&b)
}

fmt.Fprintf(&b, "## %s\n\n", i18n.T("report_by_agent"))
fmt.Fprintf(&b, "| %s | %s | %s |\n|---|---:|---:|\n", i18n.T("report_agent"), i18n.T("report_sessions"), i18n.T("report_cost"))
type akv struct {
Expand Down Expand Up @@ -630,6 +651,26 @@ func ReportOverviewHTML(ov Overview, sessions []Session) string {
w(fmt.Sprintf(`<section><h2>%s</h2><p>%s</p></section>`, html.EscapeString(i18n.T("trend_title")), html.EscapeString(trend.Message)))
}

w(fmt.Sprintf(`<section><h2>%s</h2>`, html.EscapeString(i18n.T("incident_timeline_title"))))
timelines := overviewIncidentTimelines(orderedSessions, 8)
if len(timelines) == 0 {
w(fmt.Sprintf(`<p>%s</p>`, html.EscapeString(i18n.T("incident_timeline_no_evidence"))))
} else {
w(fmt.Sprintf(`<table><thead><tr><th>%s</th><th>%s</th><th>%s</th><th>%s</th></tr></thead><tbody>`,
html.EscapeString(i18n.T("report_session")), html.EscapeString(i18n.T("incident_timeline_signal")), html.EscapeString(i18n.T("incident_timeline_evidence")), html.EscapeString(i18n.T("incident_timeline_severity"))))
for _, timeline := range timelines {
for _, item := range timeline.Items {
w(fmt.Sprintf(`<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>`,
html.EscapeString(timeline.Session),
html.EscapeString(item.Label),
html.EscapeString(item.Detail),
html.EscapeString(reportSeverityLabel(item.Severity))))
}
}
w(`</tbody></table>`)
}
w(`</section>`)

w(fmt.Sprintf(`<section><h2>%s</h2><table><thead><tr><th>%s</th><th>%s</th><th>%s</th><th class="num">%s</th><th class="num">%s</th><th class="num">%s</th><th class="num">%s</th></tr></thead><tbody>`,
html.EscapeString(i18n.T("report_recent_sessions")), html.EscapeString(i18n.T("report_session")), html.EscapeString(i18n.T("report_source")), html.EscapeString(i18n.T("report_model")), html.EscapeString(i18n.T("report_total_tokens")), html.EscapeString(i18n.T("report_cost")), html.EscapeString(i18n.T("report_health")), html.EscapeString(i18n.T("report_anomalies"))))
limit := minReportInt(len(orderedSessions), 20)
Expand Down Expand Up @@ -888,6 +929,27 @@ func markdownCell(value string) string {
return value
}

func overviewIncidentTimelines(sessions []Session, limit int) []IncidentTimelineSummary {
if limit <= 0 {
return []IncidentTimelineSummary{}
}
items := make([]IncidentTimelineSummary, 0, minReportInt(len(sessions), limit))
for _, s := range sessions {
timeline := BuildIncidentTimeline(s)
if len(timeline.Items) == 0 {
continue
}
items = append(items, timeline)
if len(items) >= limit {
break
}
}
if items == nil {
return []IncidentTimelineSummary{}
}
return items
}

// LoopCostSection generates the loop cost breakdown section for text reports.
func LoopCostSection(lc LoopCost) string {
var b strings.Builder
Expand Down
Loading