diff --git a/.github/scripts/benchmark.R b/.github/scripts/benchmark.R new file mode 100644 index 000000000..72ed6da60 --- /dev/null +++ b/.github/scripts/benchmark.R @@ -0,0 +1,105 @@ +# -- 1. Load PR version -- +message("--- Loading PR version ---") +pkgload::load_all(".") +pr_version <- as.character(packageVersion("cards")) +message("PR version: ", pr_version) + +pr_ard_summary <- cards::ard_summary +pr_data <- cards::ADSL + +pr_ard_summary(data = pr_data, variables = AGE) +pkgload::unload("cards") + +# -- 2. Install and load main version -- +message("--- Installing main version ---") +remotes::install_github("insightsengineering/cards", quiet = TRUE) +library(cards) +main_version <- as.character(packageVersion("cards")) +message("Main version: ", main_version) + +main_ard_summary <- cards::ard_summary +main_data <- cards::ADSL + +# -- 3. Multi-round benchmarks -- +set.seed(42) +n_rounds <- 5L + +message("--- Running ard_summary benchmarks (", n_rounds, " rounds) ---") +bench_rounds <- lapply(seq_len(n_rounds), function(r) { + message(" Round ", r) + res <- bench::mark( + `ard_summary (main)` = main_ard_summary(data = main_data, variables = AGE), + `ard_summary (pr)` = pr_ard_summary(data = pr_data, variables = AGE), + iterations = 100, + check = FALSE + ) + data.frame( + expression = as.character(res$expression), + median_s = as.numeric(res$median), + round = r + ) +}) |> + do.call(what = rbind) + +# -- 4. Build comparison table with confidence intervals -- +build_comparison <- function(rounds_df) { + rounds_df$group <- sub(" \\((main|pr)\\)$", "", rounds_df$expression) + rounds_df$version <- ifelse( + grepl("(main)", rounds_df$expression, fixed = TRUE), "main", "pr" + ) + + groups <- unique(rounds_df$group) + rows <- lapply(groups, function(g) { + main_medians <- rounds_df$median_s[rounds_df$group == g & rounds_df$version == "main"] + pr_medians <- rounds_df$median_s[rounds_df$group == g & rounds_df$version == "pr"] + + ratios <- pr_medians / main_medians + mean_ratio <- mean(ratios) + diff_pct <- (mean_ratio - 1) * 100 + + n <- length(ratios) + se <- sd(ratios) / sqrt(n) + t_crit <- qt(0.975, df = n - 1) + ci_lo <- (mean_ratio - t_crit * se - 1) * 100 + ci_hi <- (mean_ratio + t_crit * se - 1) * 100 + + if (ci_hi < 0) { + verdict <- paste0("\U2705 ", round(diff_pct, 1), "%") + } else if (ci_lo > 0) { + verdict <- paste0("\U274C +", round(diff_pct, 1), "%") + } else { + sign_chr <- ifelse(diff_pct >= 0, "+", "") + verdict <- paste0("\U2796 ", sign_chr, round(diff_pct, 1), "%") + } + + data.frame( + expression = g, + main = paste0(round(mean(main_medians) * 1000, 2), "ms"), + pr = paste0(round(mean(pr_medians) * 1000, 2), "ms"), + change = verdict, + ci = paste0("[", round(ci_lo, 1), "%, ", round(ci_hi, 1), "%]") + ) + }) + do.call(rbind, rows) +} + +bench_tab <- build_comparison(bench_rounds) + +header <- paste0( + "## Performance Benchmark\n\n", + "Comparing **main** (`", main_version, "`) vs **PR** (`", pr_version, "`)\n\n", + "Each benchmark runs ", n_rounds, " independent rounds. ", + "The **change** column shows the mean % difference (negative = faster).\n", + "The **95% CI** column shows the confidence interval on the change. ", + "If the CI excludes 0%, the result is flagged as a real improvement (\U2705) or regression (\U274C).\n\n" +) + +bench_section <- paste0( + "### ard_summary()\n\n", + paste(knitr::kable(bench_tab, format = "markdown"), collapse = "\n"), + "\n" +) + +report <- paste0(header, bench_section) +writeLines(report, "bench_report.md") +cat(report) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 6a9cd5dbd..c024cd1e1 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -31,8 +31,8 @@ jobs: - {os: ubuntu-latest, r: 'release'} - {os: ubuntu-latest, r: 'oldrel-1'} - {os: ubuntu-latest, r: 'oldrel-2'} - # - {os: ubuntu-latest, r: 'oldrel-3'} - # - {os: ubuntu-latest, r: 'oldrel-4'} + - {os: ubuntu-latest, r: 'oldrel-3'} + - {os: ubuntu-latest, r: 'oldrel-4'} env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml new file mode 100644 index 000000000..b400ac832 --- /dev/null +++ b/.github/workflows/benchmark.yaml @@ -0,0 +1,41 @@ +name: Performance Benchmark + +on: + pull_request: + branches: [main] + +jobs: + benchmark: + if: startsWith(github.event.pull_request.title, 'perf') + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + permissions: + pull-requests: write + steps: + - name: Checkout PR + uses: actions/checkout@v4 + + - name: Setup R + uses: r-lib/actions/setup-r@v2 + + - name: Install Dependencies & Cache + uses: r-lib/actions/setup-r-dependencies@v2 + with: + dependencies: '"hard"' + extra-packages: | + any::bench + any::knitr + any::remotes + any::pkgload + cache: true + cache-version: 1 + + - name: Run Benchmarks + run: Rscript .github/scripts/benchmark.R + + - name: Post Comment + uses: mshick/add-pr-comment@v2 + with: + message-path: bench_report.md + refresh-main-comment: true