Skip to content

Aggregated Metrics -average coverage, number of crashes, and execution time. #1005

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open
3 changes: 3 additions & 0 deletions report/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ class Benchmark:
project: str = ''
function: str = ''
language: str = ''
accumulated_results: AccumulatedResult = dataclasses.field(
default_factory=AccumulatedResult)
execution_time: float = 0.0


@dataclasses.dataclass
Expand Down
7 changes: 7 additions & 0 deletions report/templates/index.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,13 @@
{% if benchmark.result.max_coverage_diff_report %}
,"max_line_coverage_diff_report": "{{ benchmark.result.max_coverage_diff_report }}"
{%- endif %}
,"accumulated_results": {
"total_runs": {{ benchmark.accumulated_results.total_runs }},
"total_coverage": {{ benchmark.accumulated_results.total_coverage }},
"crashes": {{ benchmark.accumulated_results.crashes }},
"crash_cases": {{ benchmark.accumulated_results.crash_cases }}
}
,"execution_time": {{ benchmark.execution_time }}
}{% if not loop.last %},{% endif %}
{% endfor %}
]
Expand Down
1 change: 1 addition & 0 deletions report/trends_report_web/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ <h3>Tags</h3>
<h2>Overview</h2>
<div id="overview-chart" class="chart"></div>
<div id="overview-table"></div>
<div id="aggregated-metrics" class="aggregated-metrics"></div>
<h2>Projects</h2>
<div id="overview-coverage-chart" class="chart"></div>
<div id="projects" class="projects"></div>
Expand Down
75 changes: 75 additions & 0 deletions report/trends_report_web/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,74 @@ class Page {
container.append(chart);
}

// ── new method to sum/average across all filtered experiments ─────────────
computeAggregates() {
let totalRuns = 0;
let totalCoverage = 0;
let totalCrashes = 0;
let totalCrashCases = 0;
let totalExecTime = 0;

for (let name of this.filteredNames) {
const rpt = this.reports.get(name).accumulated_results;
totalRuns += rpt.total_runs;
totalCoverage += rpt.total_coverage;
totalCrashes += rpt.crashes;
totalCrashCases += rpt.crash_cases;
totalExecTime += (this.reports.get(name).execution_time || 0);
}

return {
avgCoverage: totalRuns
? (totalCoverage / totalRuns) * 100
: 0,
crashes: totalCrashes,
crashCases: totalCrashCases,
avgExecTime: this.filteredNames.length
? (totalExecTime / this.filteredNames.length)
: 0
};
}

// ── Build the separate "Aggregated Metrics 📊" table ─────────────
updateAggregatedMetrics() {
const agg = this.computeAggregates();
const container = document.querySelector('#aggregated-metrics');
container.replaceChildren();

// Heading
const title = document.createElement('h3');
title.innerText = 'Aggregated Metrics 📊';
container.appendChild(title);

// Table
const table = document.createElement('table');
const thead = document.createElement('thead');
const tbody = document.createElement('tbody');
table.appendChild(thead);
table.appendChild(tbody);
container.appendChild(table);

// Header row
const headerRow = document.createElement('tr');
for (let h of ['Total Crashes', 'Total Crash Cases', 'Average Coverage', 'Execution Time']) {
const th = document.createElement('th');
th.innerText = h;
headerRow.appendChild(th);
}
thead.appendChild(headerRow);

// Data row
const dataRow = document.createElement('tr');
dataRow.innerHTML = `
<td>${agg.crashes}</td>
<td>${agg.crashCases}</td>
<td>${agg.avgCoverage.toFixed(2)}%</td>
<td>${agg.avgExecTime.toFixed(1)}s</td>
`;
tbody.appendChild(dataRow);
}

// updateOverviewTable updates the data table showing the coverage data for
// the selected project.
updateOverviewTable() {
Expand All @@ -361,8 +429,12 @@ class Page {
const table = document.createElement('table');
const thead = document.createElement('thead');
const tbody = document.createElement('tbody');
const tfoot = document.createElement('tfoot');
tfoot.id = 'overview-aggregate';

table.appendChild(thead);
table.appendChild(tbody);
table.appendChild(tfoot);
tableContainer.appendChild(table);


Expand Down Expand Up @@ -409,6 +481,9 @@ class Page {
lineCoverageDiff.innerText = `${lineCoverageDiffVal}%`;
tr.appendChild(lineCoverageDiff);
}

// ── inject the Total/Avg row into our new <tfoot> ─────────────
this.updateAggregatedMetrics();
}

// updateOverviewCoverageChart configures Plot to chart the coverage gain of different
Expand Down
20 changes: 20 additions & 0 deletions report/trends_report_web/style.css
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,26 @@ button {
margin-left: 5px;
}

/*
*aggregated metrics
*/

#aggregated-metrics {
margin: 1rem 0;
}
#aggregated-metrics table {
width: auto;
border-collapse: collapse;
}
#aggregated-metrics th,
#aggregated-metrics td {
padding: 0.5rem 1rem;
border: 1px solid #ddd;
}
#aggregated-metrics h3 {
margin-bottom: 0.5rem;
}

/*
* Projects
*/
Expand Down
31 changes: 26 additions & 5 deletions report/web.py
Original file line number Diff line number Diff line change
Expand Up @@ -191,15 +191,30 @@ def generate(self):
sample_targets = self._results.get_targets(benchmark.id, sample.id)
self._write_benchmark_sample(benchmark, sample, sample_targets)

time_results = self.read_timings()

# 3) Enrich each benchmark with its own aggregated metrics and execution time
for bm in benchmarks:
# a) per-benchmark accumulated results
bm.accumulated_results = self._results.get_macro_insights([bm])
# b) parse total_run_time "HH:MM:SS(.sss)" into seconds
rt_str = time_results.get('total_run_time', '0:00:00')
try:
h, m, s = rt_str.split(':')
bm.execution_time = int(h) * 3600 + int(m) * 60 + float(s)
except Exception:
bm.execution_time = 0.0

accumulated_results = self._results.get_macro_insights(benchmarks)
projects = self._results.get_project_summary(benchmarks)
coverage_language_gains = self._results.get_coverage_language_gains()

time_results = self.read_timings()

self._write_index_html(benchmarks, accumulated_results, time_results,
projects, samples_with_bugs, coverage_language_gains)
self._write_index_json(benchmarks)
self._write_index_json(benchmarks,
accumulated_results=accumulated_results,
time_results=time_results,
projects=projects)

def _write(self, output_path: str, content: str):
"""Utility write to filesystem function."""
Expand Down Expand Up @@ -232,9 +247,15 @@ def _write_index_html(self, benchmarks: List[Benchmark],
coverage_language_gains=coverage_language_gains)
self._write('index.html', rendered)

def _write_index_json(self, benchmarks: List[Benchmark]):
def _write_index_json(self, benchmarks: List[Benchmark],
accumulated_results: AccumulatedResult,
time_results: dict[str, Any], projects: list[Project]):
"""Generate the report index.json and write to filesystem."""
rendered = self._jinja.render('index.json', benchmarks=benchmarks)
rendered = self._jinja.render('index.json',
benchmarks=benchmarks,
accumulated_results=accumulated_results,
time_results=time_results,
projects=projects)
self._write('index.json', rendered)

def _write_benchmark_index(self, benchmark: Benchmark, samples: List[Sample],
Expand Down