perf(harness): measure first hot calls of fresh bytecode functions

Author: eval-exec

crates/neomacs-perf/fixtures/first-hot-loop.el

@@ -0,0 +1,129 @@
+;;; first-hot-loop.el --- first calls through hot loops -*- lexical-binding: t; -*-
+(require 'bytecomp)
+(require 'json)
+
+;; One operation is a fresh function's first call. Byte compilation and GC are
+;; setup; interpreter heat-up, OSR compilation, execution and result collection
+;; are timed together. No function is called during preparation.
+(defconst neomacs-perf-first-hot-loop--inner-iterations 65536)
+
+(defvar neomacs-perf-first-hot-loop--profile-gate-process nil)
+(defvar neomacs-perf-first-hot-loop--profile-gate-response "")
+
+(defun neomacs-perf-first-hot-loop--profile-gate-filter (_process output)
+  (setq neomacs-perf-first-hot-loop--profile-gate-response
+        (concat neomacs-perf-first-hot-loop--profile-gate-response output)))
+
+(defun neomacs-perf-first-hot-loop--profile-gate-connect ()
+  (let* ((port-text (getenv "NEOMACS_PERF_GATE_PORT"))
+         (port (and port-text (string-to-number port-text))))
+    (when (and port-text (not (> port 0)))
+      (error "invalid first-hot-loop profile gate port %S" port-text))
+    (when (and port-text
+               (not (process-live-p
+                     neomacs-perf-first-hot-loop--profile-gate-process)))
+      (setq neomacs-perf-first-hot-loop--profile-gate-process
+            (make-network-process
+             :name "neomacs-perf-first-hot-loop-gate"
+             :family 'ipv4
+             :host "127.0.0.1"
+             :service port
+             :coding 'binary
+             :noquery t
+             :filter #'neomacs-perf-first-hot-loop--profile-gate-filter)))
+    neomacs-perf-first-hot-loop--profile-gate-process))
+
+(defun neomacs-perf-first-hot-loop--sampling-command (command)
+  (let ((process (neomacs-perf-first-hot-loop--profile-gate-connect)))
+    (when process
+      (setq neomacs-perf-first-hot-loop--profile-gate-response "")
+      (process-send-string process (concat command "\n"))
+      (let ((deadline (+ (float-time) 30.0)))
+        (while (and
+                (not (and
+                      (> (length
+                          neomacs-perf-first-hot-loop--profile-gate-response)
+                         0)
+                      (= (aref
+                          neomacs-perf-first-hot-loop--profile-gate-response
+                          (1- (length
+                               neomacs-perf-first-hot-loop--profile-gate-response)))
+                         ?\n)))
+                (< (float-time) deadline))
+          (unless (process-live-p process)
+            (error "first-hot-loop profile gate disconnected during %s" command))
+          (accept-process-output process 0.05))
+        (unless (equal neomacs-perf-first-hot-loop--profile-gate-response
+                       "ack\n")
+          (error "first-hot-loop profile gate rejected %s: %S"
+                 command
+                 neomacs-perf-first-hot-loop--profile-gate-response))))))
+
+(defun neomacs-perf-first-hot-loop--close-profile-gate ()
+  (when (processp neomacs-perf-first-hot-loop--profile-gate-process)
+    (delete-process neomacs-perf-first-hot-loop--profile-gate-process)
+    (setq neomacs-perf-first-hot-loop--profile-gate-process nil)))
+
+(defun neomacs-perf-first-hot-loop--run ()
+  (let* ((scenario (getenv "NEOMACS_PERF_WORKLOAD"))
+         (iterations (string-to-number (or (getenv "NEOMACS_PERF_ITERATIONS") "0")))
+         (functions nil) (results nil) (index 0) (compiled t)
+         (prepared 0) (completed 0) (elapsed-us 0) (wall-us 0)
+         (status "error") (error-message nil) (exit-code 2))
+    (condition-case err
+        (progn
+          (unless (and (> iterations 0) (equal scenario "first-hot-loop"))
+            (error "Invalid first-hot-loop input"))
+          (while (< index iterations)
+            ;; Fresh compiler outputs and distinct constants prevent accidental
+            ;; reuse of one function's native cache for the entire sample.
+            (let ((function
+                   (byte-compile
+                    `(lambda (n)
+                       (let ((held ,index) (i 0) (sum 0))
+                         (while (< i n)
+                           (setq sum (+ sum i) i (1+ i)))
+                         (list i sum held))))))
+              (unless (byte-code-function-p function)
+                (setq compiled nil)
+                (error "First-hot-loop function %d is not bytecode" index))
+              (push function functions))
+            (setq index (1+ index)))
+          (setq functions (nreverse functions) prepared (length functions))
+          (garbage-collect)
+          (neomacs-perf-first-hot-loop--sampling-command "enable")
+          (unwind-protect
+              (let ((cpu-start (car (current-cpu-time)))
+                    (wall-start (float-time)))
+                (dolist (function functions)
+                  (push (funcall function neomacs-perf-first-hot-loop--inner-iterations)
+                        results))
+                (setq elapsed-us (- (car (current-cpu-time)) cpu-start)
+                      wall-us (round (* 1000000 (- (float-time) wall-start)))))
+            (neomacs-perf-first-hot-loop--sampling-command "disable"))
+          ;; Keep every result for independent host validation after timing.
+          (setq results (nreverse results) completed (length results)
+                status "ok" exit-code 0))
+      (error (setq error-message (error-message-string err))))
+    (neomacs-perf-first-hot-loop--close-profile-gate)
+    (with-temp-file (getenv "NEOMACS_PERF_RESULT")
+      (insert
+       (json-serialize
+        `((schema_version . 1) (scenario . ,scenario) (status . ,status)
+          (iterations . ,iterations)
+          (inner_iterations . ,neomacs-perf-first-hot-loop--inner-iterations)
+          (prepared_functions . ,prepared)
+          (bytecode_compiled . ,(if compiled t :json-false))
+          (completed_operations . ,completed)
+          (results . ,(vconcat (mapcar #'vconcat results)))
+          (elapsed_us . ,elapsed-us) (elapsed_wall_us . ,wall-us)
+          (error . ,error-message))
+        :false-object :json-false :null-object nil)))
+    (write-region "done\n" nil (getenv "SENTINEL") nil 'silent)
+    (kill-emacs exit-code)))
+
+(if noninteractive
+    (neomacs-perf-first-hot-loop--run)
+  (run-at-time 0 nil #'neomacs-perf-first-hot-loop--run))
+
+;;; first-hot-loop.el ends here

crates/neomacs-perf/src/catalog.rs

@@ -38,6 +38,8 @@ pub enum ScenarioId {
     /// Rare-call tier-entry diagnostics, excluded from the whole-editor suite.
     LexicalLoop,
     DynamicBindingLoop,
+    /// First call of fresh functions, including heat-up and OSR compilation.
+    FirstHotLoop,
     EditingSimulation,
     Startup,
     SustainedEditing,
@@ -150,6 +152,7 @@ impl ScenarioId {
             Self::BytecodeCallLoop => "bytecode-call-loop",
             Self::LexicalLoop => "lexical-loop",
             Self::DynamicBindingLoop => "dynamic-binding-loop",
+            Self::FirstHotLoop => "first-hot-loop",
             Self::EditingSimulation => "editing-simulation",
             Self::Startup => "startup",
             Self::SustainedEditing => "sustained-editing",
@@ -206,6 +209,7 @@ impl FromStr for ScenarioId {
             "bytecode-call-loop" => Ok(Self::BytecodeCallLoop),
             "lexical-loop" => Ok(Self::LexicalLoop),
             "dynamic-binding-loop" => Ok(Self::DynamicBindingLoop),
+            "first-hot-loop" => Ok(Self::FirstHotLoop),
             "editing-simulation" => Ok(Self::EditingSimulation),
             "startup" => Ok(Self::Startup),
             "sustained-editing" => Ok(Self::SustainedEditing),
@@ -526,6 +530,14 @@ const SCENARIOS: &[ScenarioSpec] = &[
         primary_metric: MetricName::PerOperationWallTime,
         cross_editor_parity_metrics: &[],
     },
+    ScenarioSpec {
+        id: ScenarioId::FirstHotLoop,
+        description: "First call of fresh bytecode functions, each looping 65,536 times under the editor default tier policy",
+        default_frontend: Frontend::Batch,
+        default_iterations: NonZeroU32::new(100).expect("non-zero scenario default"),
+        primary_metric: MetricName::PerOperationWallTime,
+        cross_editor_parity_metrics: &[],
+    },
 ];
 
 pub fn scenarios() -> &'static [ScenarioSpec] {
@@ -544,6 +556,7 @@ pub const fn scenario(id: ScenarioId) -> &'static ScenarioSpec {
         ScenarioId::BytecodeCallLoop => &SCENARIOS[2],
         ScenarioId::LexicalLoop => &SCENARIOS[27],
         ScenarioId::DynamicBindingLoop => &SCENARIOS[28],
+        ScenarioId::FirstHotLoop => &SCENARIOS[29],
         ScenarioId::EditingSimulation => &SCENARIOS[3],
         ScenarioId::Startup => &SCENARIOS[4],
         ScenarioId::SustainedEditing => &SCENARIOS[5],

crates/neomacs-perf/src/catalog_test.rs

@@ -7,7 +7,7 @@ use super::{CrossEditorParityMetric, Frontend, MetricName, ScenarioId, scenario,
 #[test]
 fn catalog_exposes_the_rust_lsp_typing_workload_as_a_typed_scenario() {
     let scenarios = scenarios();
-    assert_eq!(scenarios.len(), 29);
+    assert_eq!(scenarios.len(), 30);
 
     // The heavy row exists so the light one keeps its baseline: same workload,
     // a whole-file diagnostic set instead of four on adjacent lines.
@@ -300,12 +300,23 @@ fn bounded_search_diagnostics_are_portable_and_stay_out_of_the_editor_score() {
 
 #[test]
 fn rare_call_loop_diagnostics_stay_out_of_editor_score() {
-    for id in [ScenarioId::LexicalLoop, ScenarioId::DynamicBindingLoop] {
+    for id in [
+        ScenarioId::LexicalLoop,
+        ScenarioId::DynamicBindingLoop,
+        ScenarioId::FirstHotLoop,
+    ] {
         assert_eq!(ScenarioId::from_str(id.as_str()), Ok(id));
         let spec = scenario(id);
         assert_eq!(spec.id, id);
         assert_eq!(spec.default_frontend, Frontend::Batch);
-        assert_eq!(spec.default_iterations.get(), 1_000_000);
+        assert_eq!(
+            spec.default_iterations.get(),
+            if id == ScenarioId::FirstHotLoop {
+                100
+            } else {
+                1_000_000
+            }
+        );
         assert_eq!(spec.primary_metric, MetricName::PerOperationWallTime);
         assert!(
             !crate::suite::SuiteId::Standard

crates/neomacs-perf/src/harness.rs

@@ -514,7 +514,7 @@ impl PerfHarness {
             | ScenarioId::BoundedSearchNoEdit => {
                 scenarios::bounded_search::prepare(&self.workspace_root, request, run_directory)
             }
-            ScenarioId::LexicalLoop | ScenarioId::DynamicBindingLoop => {
+            ScenarioId::LexicalLoop | ScenarioId::DynamicBindingLoop | ScenarioId::FirstHotLoop => {
                 scenarios::vm_loop::prepare(&self.workspace_root, request, run_directory)
             }
             ScenarioId::BytecodeCallLoop => {
@@ -1582,6 +1582,7 @@ enum ScenarioResult {
     MxTabCompletion(scenarios::mx_tab::MxTabCompletionResult),
     BytecodeCallLoop(scenarios::bytecode::BytecodeCallLoopResult),
     VmLoop(scenarios::vm_loop::VmLoopResult),
+    FirstHotLoop(scenarios::vm_loop::first_entry::FirstHotLoopResult),
     BoundedSearch(scenarios::bounded_search::BoundedSearchResult),
     ElispBenchmarks(scenarios::elisp_benchmarks::ElispBenchmarksResult),
     EditorWorkload(scenarios::editor_workload::EditorWorkloadResult),
@@ -1604,6 +1605,7 @@ impl ScenarioResult {
             Self::MxTabCompletion(result) => result.elapsed_us,
             Self::BytecodeCallLoop(result) => result.elapsed_us,
             Self::VmLoop(result) => result.elapsed_us(),
+            Self::FirstHotLoop(result) => result.elapsed_us(),
             Self::BoundedSearch(result) => result.elapsed_us(),
             Self::ElispBenchmarks(result) => result.elapsed_us,
             Self::EditorWorkload(result) => result.elapsed_us,
@@ -1630,6 +1632,7 @@ fn parse_scenario_result(
         | ScenarioId::BoundedSearchNoEdit => {
             serde_json::from_str(raw).map(ScenarioResult::BoundedSearch)
         }
+        ScenarioId::FirstHotLoop => serde_json::from_str(raw).map(ScenarioResult::FirstHotLoop),
         ScenarioId::LexicalLoop | ScenarioId::DynamicBindingLoop => {
             serde_json::from_str(raw).map(ScenarioResult::VmLoop)
         }
@@ -1853,6 +1856,9 @@ fn result_verdict(
     process_wall_us: u128,
 ) -> RunVerdict {
     let mismatches = match result {
+        ScenarioResult::FirstHotLoop(result) => {
+            scenarios::vm_loop::first_entry::validate(request, result)
+        }
         ScenarioResult::VmLoop(result) => {
             scenarios::vm_loop::validate_vm_loop_result(request, result)
         }
@@ -1907,6 +1913,9 @@ where
 
 fn valid_measurements(result: &ScenarioResult, wall_elapsed_us: u128) -> Vec<Measurement> {
     match result {
+        ScenarioResult::FirstHotLoop(result) => {
+            scenarios::vm_loop::first_entry::measurements(result, wall_elapsed_us)
+        }
         ScenarioResult::VmLoop(result) => {
             scenarios::vm_loop::valid_vm_loop_measurements(result, wall_elapsed_us)
         }

crates/neomacs-perf/src/harness/scenarios/editor_workload.rs

@@ -665,7 +665,8 @@ pub(crate) fn validate_editor_workload_result(
         | ScenarioId::MxTabCompletion
         | ScenarioId::BytecodeCallLoop
         | ScenarioId::LexicalLoop
-        | ScenarioId::DynamicBindingLoop => {
+        | ScenarioId::DynamicBindingLoop
+        | ScenarioId::FirstHotLoop => {
             unreachable!("dedicated scenario results do not use the editor workload validator")
         }
     }

crates/neomacs-perf/src/harness/scenarios/vm_loop.rs

@@ -1,5 +1,8 @@
 //! Rarely called, long-running bytecode loops under the editor's normal tier policy.
 //! The paired scenarios differ by a live special binding across the hot loop.
+//! First-entry diagnostics share preparation but validate every fresh function.
+
+pub(crate) mod first_entry;
 
 use std::collections::BTreeMap;
 use std::fs;
@@ -23,14 +26,19 @@ pub(crate) fn prepare(
 ) -> Result<PreparedScenario, String> {
     let sandbox = MelpaSandbox::new(&format!("perf-{}", request.scenario))?;
     let editor = collect_editor_provenance(request.editor(), &sandbox)?;
-    let fixture_source = workspace_root.join("crates/neomacs-perf/fixtures/vm-loop.el");
+    let workload_source = if request.scenario == ScenarioId::FirstHotLoop {
+        "crates/neomacs-perf/fixtures/first-hot-loop.el"
+    } else {
+        "crates/neomacs-perf/fixtures/vm-loop.el"
+    };
+    let fixture_source = workspace_root.join(workload_source);
     if !fixture_source.is_file() {
         return Err(format!(
             "missing committed performance fixture {}",
             fixture_source.display()
         ));
     }
-    let fixture = run_directory.join("vm-loop.el");
+    let fixture = run_directory.join(fixture_source.file_name().expect("fixture filename"));
     fs::copy(&fixture_source, &fixture).map_err(|error| {
         format!(
             "failed to copy performance fixture {} to {}: {error}",
@@ -42,7 +50,7 @@ pub(crate) fn prepare(
     let provenance_manifest = VmLoopInputProvenanceManifest {
         editor,
         host: collect_host_provenance(request.machine_policy()),
-        workload_source: "crates/neomacs-perf/fixtures/vm-loop.el",
+        workload_source,
         workload_source_sha256: sha256_file(&fixture_source)?,
         execution_policy: "editor-default-with-recorded-overrides",
         environment_policy: "closed-v1",
@@ -226,23 +234,36 @@ pub(crate) fn valid_vm_loop_measurements(
     wall_elapsed_us: u128,
 ) -> Vec<Measurement> {
     let r = &result.wire;
+    loop_measurements(
+        r.iterations,
+        r.completed_operations,
+        r.elapsed_us,
+        r.elapsed_wall_us,
+        wall_elapsed_us,
+    )
+}
+
+fn loop_measurements(
+    iterations: u32,
+    completed: u32,
+    cpu_us: u64,
+    wall_us: u64,
+    process_us: u128,
+) -> Vec<Measurement> {
     [
-        (MetricName::ProcessWallTime, wall_elapsed_us as f64),
-        (MetricName::WorkloadCpuTime, r.elapsed_us as f64),
-        (MetricName::WorkloadWallTime, r.elapsed_wall_us as f64),
+        (MetricName::ProcessWallTime, process_us as f64),
+        (MetricName::WorkloadCpuTime, cpu_us as f64),
+        (MetricName::WorkloadWallTime, wall_us as f64),
         (
             MetricName::PerOperationCpuTime,
-            r.elapsed_us as f64 / f64::from(r.iterations),
+            cpu_us as f64 / f64::from(iterations),
         ),
         (
             MetricName::PerOperationWallTime,
-            r.elapsed_wall_us as f64 / f64::from(r.iterations),
-        ),
-        (
-            MetricName::OperationCount,
-            f64::from(r.completed_operations),
+            wall_us as f64 / f64::from(iterations),
         ),
-        (MetricName::Iterations, f64::from(r.iterations)),
+        (MetricName::OperationCount, f64::from(completed)),
+        (MetricName::Iterations, f64::from(iterations)),
     ]
     .into_iter()
     .map(|(name, value)| Measurement {