Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 27 additions & 15 deletions internal/engine/engine_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1961,28 +1961,34 @@ func TestReportOverviewHTMLIsShareableAndEscaped(t *testing.T) {
Name: `good<script>`,
Health: 92,
Metrics: Metrics{
SourceTool: "aider",
ModelUsed: "gpt-4.1",
TokensInput: 1000,
TokensOutput: 500,
TokensCacheW: 25,
TokensCacheR: 50,
ToolCallsOK: 4,
ToolCallsFail: 1,
CostEstimated: 0.12,
SourceTool: "aider",
ModelUsed: "gpt-4.1",
TokensInput: 1000,
TokensOutput: 500,
TokensCacheW: 25,
TokensCacheR: 50,
ToolCallsOK: 4,
ToolCallsFail: 1,
ToolUsage: map[string]int{"go test": 1},
ToolAuthority: map[string]int{ToolAuthorityTestOrBuild: 1},
HighestAuthority: ToolAuthorityTestOrBuild,
CostEstimated: 0.12,
},
},
{
Name: "bad",
Health: 30,
Anomalies: []Anomaly{{Type: "hanging", Severity: SeverityHigh}},
Metrics: Metrics{
SourceTool: "cursor",
ModelUsed: "default",
TokensInput: 300,
TokensOutput: 200,
ToolCallsFail: 5,
CostEstimated: 0.34,
SourceTool: "cursor",
ModelUsed: "default",
TokensInput: 300,
TokensOutput: 200,
ToolCallsFail: 5,
ToolUsage: map[string]int{"bash": 1},
ToolAuthority: map[string]int{ToolAuthorityShellExec: 1},
HighestAuthority: ToolAuthorityShellExec,
CostEstimated: 0.34,
},
},
}
Expand All @@ -1994,6 +2000,12 @@ func TestReportOverviewHTMLIsShareableAndEscaped(t *testing.T) {
"Total tokens",
"<strong>2075</strong>",
"Tool failures",
"Tool authority",
"Highest category",
"Authority category counts",
"High-authority tools",
"<code>shell_exec</code>",
"<code>bash</code>",
"Aider",
"Cursor",
"good&lt;script&gt;",
Expand Down
71 changes: 71 additions & 0 deletions internal/engine/report.go
Original file line number Diff line number Diff line change
Expand Up @@ -632,6 +632,7 @@ func ReportOverviewMarkdown(ov Overview, sessions []Session) string {
func ReportOverviewHTML(ov Overview, sessions []Session) string {
orderedSessions := canonicalOverviewSessions(sessions)
summary := overviewReportSummary(orderedSessions)
authority := overviewAuthoritySummary(orderedSessions)
trend := AnalyzeHealthTrend(orderedSessions)
agents := sortedAgents(ov.ByAgent)
models := sortedModels(ov.ByModel)
Expand Down Expand Up @@ -667,6 +668,24 @@ func ReportOverviewHTML(ov Overview, sessions []Session) string {
w(fmt.Sprintf(`<div class="metric"><span>%s</span><strong>$%.2f</strong><p>%s</p></div>`, html.EscapeString(i18n.T("total_cost")), ov.TotalCost, html.EscapeString(i18n.T("report_estimated_cost"))))
w(fmt.Sprintf(`<div class="metric %s"><span>%s</span><strong>%d/%d</strong><p>%s</p></div>`, html.EscapeString(failureClass(summary.ToolFailRate)), html.EscapeString(i18n.T("report_tool_failures")), summary.FailedTools, summary.TotalTools, html.EscapeString(fmt.Sprintf(i18n.T("report_failure_rate"), summary.ToolFailRate))))
w(`</div>`)
if authority.HasData {
w(fmt.Sprintf(`<section><h2>%s</h2>`, html.EscapeString(i18n.T("report_tool_authority"))))
if authority.Highest != "" {
w(fmt.Sprintf(`<p><strong>%s</strong>: <code>%s</code></p>`, html.EscapeString(i18n.T("report_highest_authority")), html.EscapeString(authority.Highest)))
}
if len(authority.Counts) > 0 {
w(fmt.Sprintf(`<table><caption>%s</caption><thead><tr><th>%s</th><th class="num">%s</th></tr></thead><tbody>`,
html.EscapeString(i18n.T("report_authority_category_counts")), html.EscapeString(i18n.T("report_authority_category")), html.EscapeString(i18n.T("report_count"))))
for _, item := range authority.Counts {
w(fmt.Sprintf(`<tr><td><code>%s</code></td><td class="num">%d</td></tr>`, html.EscapeString(item.Category), item.Count))
}
w(`</tbody></table>`)
}
if len(authority.HighTools) > 0 {
w(fmt.Sprintf(`<p><strong>%s</strong>: %s</p>`, html.EscapeString(i18n.T("report_high_authority_tools")), reportHTMLCodeList(authority.HighTools)))
}
w(`</section>`)
}
if len(orderedSessions) > 1 {
w(fmt.Sprintf(`<section><h2>%s</h2><p>%s</p></section>`, html.EscapeString(i18n.T("trend_title")), html.EscapeString(trend.Message)))
}
Expand Down Expand Up @@ -789,6 +808,18 @@ type overviewSummary struct {
ToolFailRate float64
}

type authorityCount struct {
Category string
Count int
}

type overviewAuthority struct {
Highest string
Counts []authorityCount
HighTools []string
HasData bool
}

func overviewReportSummary(sessions []Session) overviewSummary {
var summary overviewSummary
totalHealth := 0
Expand All @@ -807,6 +838,36 @@ func overviewReportSummary(sessions []Session) overviewSummary {
return summary
}

func overviewAuthoritySummary(sessions []Session) overviewAuthority {
counts := make(map[string]int)
toolSurface := make(map[string]struct{})
highest := ""
for _, s := range sessions {
for tool := range s.Metrics.ToolUsage {
toolSurface[tool] = struct{}{}
}
for category, count := range s.Metrics.ToolAuthority {
if count > 0 {
counts[category] += count
highest = HigherToolAuthority(highest, category)
}
}
highest = HigherToolAuthority(highest, s.Metrics.HighestAuthority)
}
keys := sortedReportIntKeys(counts)
items := make([]authorityCount, 0, len(keys))
for _, key := range keys {
items = append(items, authorityCount{Category: key, Count: counts[key]})
}
highTools := highAuthorityTools(sortedReportKeys(toolSurface))
return overviewAuthority{
Highest: highest,
Counts: items,
HighTools: highTools,
HasData: highest != "" || len(items) > 0 || len(highTools) > 0,
}
}

func canonicalOverviewSessions(sessions []Session) []Session {
ordered := append([]Session(nil), sessions...)
sort.SliceStable(ordered, func(i, j int) bool {
Expand Down Expand Up @@ -952,6 +1013,16 @@ func highAuthorityTools(tools []string) []string {
return out
}

func reportHTMLCodeList(values []string) string {
parts := make([]string, 0, len(values))
for _, value := range values {
if value != "" {
parts = append(parts, fmt.Sprintf(`<code>%s</code>`, html.EscapeString(value)))
}
}
return strings.Join(parts, ", ")
}

func markdownCell(value string) string {
value = strings.ReplaceAll(value, "|", "\\|")
value = strings.ReplaceAll(value, "\n", "<br>")
Expand Down
24 changes: 24 additions & 0 deletions internal/i18n/i18n.go
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,30 @@ var translations = map[string]map[Lang]string{
EN: "Tool failures",
ZH: "工具失败",
},
"report_tool_authority": {
EN: "Tool authority",
ZH: "工具权限",
},
"report_highest_authority": {
EN: "Highest category",
ZH: "最高权限类别",
},
"report_authority_category_counts": {
EN: "Authority category counts",
ZH: "权限类别计数",
},
"report_authority_category": {
EN: "Authority category",
ZH: "权限类别",
},
"report_high_authority_tools": {
EN: "High-authority tools",
ZH: "高权限工具",
},
"report_count": {
EN: "Count",
ZH: "数量",
},
"report_failure_rate": {
EN: "%.1f%% failure rate",
ZH: "%.1f%% 失败率",
Expand Down
6 changes: 6 additions & 0 deletions scripts/ci/check-report-semantics.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@ grep -q "<div class=\"meta\">v$version" "$out_dir/semantics/overview.html" \
|| fail "html overview missing current version metadata"
grep -q "Estimated session cost" "$out_dir/semantics/overview.html" \
|| fail "html overview missing cost helper text"
grep -q "Tool authority" "$out_dir/semantics/overview.html" \
|| fail "html overview missing tool authority summary"
grep -q "Authority category counts" "$out_dir/semantics/overview.html" \
|| fail "html overview missing authority category counts"
grep -q "<code>test_or_build</code>" "$out_dir/semantics/overview.html" \
|| fail "html overview missing highest demo authority category"

node -e '
const fs = require("fs");
Expand Down