Instrument full Neo VM benchmarking coverage for opcodes, syscalls, and native methods

.gitignore

@@ -267,3 +267,4 @@ launchSettings.json
 # Benchmarks
 **/BenchmarkDotNet.Artifacts/
 /src/Neo.CLI/neo-cli/
+.serena/

benchmarks/Neo.VM.Benchmarks/Neo.VM.Benchmarks.csproj

@@ -9,6 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\..\src\Neo\Neo.csproj" />
     <ProjectReference Include="..\..\src\Neo.Extensions\Neo.Extensions.csproj" />
     <ProjectReference Include="..\..\src\Neo.Json\Neo.Json.csproj" />
     <ProjectReference Include="..\..\src\Neo.VM\Neo.VM.csproj" />

benchmarks/Neo.VM.Benchmarks/OpCode/BenchmarkEngine.cs

@@ -9,7 +9,9 @@
 // Redistribution and use in source and binary forms with or without
 // modifications are permitted.
 
+using Neo.VM.Benchmark.Infrastructure;
 using Neo.VM.Types;
+using System;
 using System.Diagnostics;
 using System.Runtime.CompilerServices;
 
@@ -22,7 +24,11 @@ public class BenchmarkEngine : ExecutionEngine
     {
         private readonly Dictionary<VM.OpCode, (int Count, TimeSpan TotalTime)> _opcodeStats = new();
         private readonly Dictionary<Script, HashSet<uint>> _breakPoints = new();
-        private long _gasConsumed = 0;
+        private long _gasConsumed;
+
+        public BenchmarkResultRecorder? Recorder { get; set; }
+        public Action<BenchmarkEngine, VM.Instruction>? BeforeInstruction { get; set; }
+        public Action<BenchmarkEngine, VM.Instruction>? AfterInstruction { get; set; }
 
         public BenchmarkEngine() : base(ComposeJumpTable()) { }
 
@@ -70,14 +76,8 @@ public void ExecuteBenchmark()
         {
             while (State != VMState.HALT && State != VMState.FAULT)
             {
-#if DEBUG
-                var stopwatch = Stopwatch.StartNew();
-#endif
-                ExecuteNext();
-#if DEBUG
-                stopwatch.Stop();
-                UpdateOpcodeStats(CurrentContext!.CurrentInstruction!.OpCode, stopwatch.Elapsed);
-#endif
+                var instruction = CurrentContext!.CurrentInstruction ?? VM.Instruction.RET;
+                ExecuteStep(instruction);
             }
 #if DEBUG
             PrintOpcodeStats();
@@ -87,76 +87,25 @@ public void ExecuteBenchmark()
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void ExecuteOneGASBenchmark()
         {
-            while (State != VMState.HALT && State != VMState.FAULT)
-            {
-                var instruction = CurrentContext!.CurrentInstruction ?? VM.Instruction.RET;
-                _gasConsumed += Benchmark_Opcode.OpCodePrices[instruction.OpCode];
-                if (_gasConsumed >= Benchmark_Opcode.OneGasDatoshi)
-                {
-                    State = VMState.HALT;
-                }
-#if DEBUG
-                var stopwatch = Stopwatch.StartNew();
-#endif
-                ExecuteNext();
-#if DEBUG
-                stopwatch.Stop();
-                UpdateOpcodeStats(instruction.OpCode, stopwatch.Elapsed);
-#endif
-            }
-#if DEBUG
-            PrintOpcodeStats();
-#endif
+            ExecuteWithGasBudget(Benchmark_Opcode.OneGasDatoshi);
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void ExecuteTwentyGASBenchmark()
         {
-            while (State != VMState.HALT && State != VMState.FAULT)
-            {
-                var instruction = CurrentContext!.CurrentInstruction ?? VM.Instruction.RET;
-                _gasConsumed += Benchmark_Opcode.OpCodePrices[instruction.OpCode];
-                if (_gasConsumed >= 20 * Benchmark_Opcode.OneGasDatoshi)
-                {
-                    State = VMState.HALT;
-                }
-#if DEBUG
-                var stopwatch = Stopwatch.StartNew();
-#endif
-                ExecuteNext();
-#if DEBUG
-                stopwatch.Stop();
-                UpdateOpcodeStats(instruction.OpCode, stopwatch.Elapsed);
-#endif
-            }
-#if DEBUG
-            PrintOpcodeStats();
-#endif
+            ExecuteWithGasBudget(20 * Benchmark_Opcode.OneGasDatoshi);
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void ExecuteOpCodesBenchmark()
         {
-            while (State != VMState.HALT && State != VMState.FAULT)
-            {
-                var instruction = CurrentContext!.CurrentInstruction ?? VM.Instruction.RET;
-                _gasConsumed += Benchmark_Opcode.OpCodePrices[instruction.OpCode];
-                if (_gasConsumed >= Benchmark_Opcode.OneGasDatoshi)
-                {
-                    State = VMState.HALT;
-                }
-#if DEBUG
-                var stopwatch = Stopwatch.StartNew();
-#endif
-                ExecuteNext();
-#if DEBUG
-                stopwatch.Stop();
-                UpdateOpcodeStats(instruction.OpCode, stopwatch.Elapsed);
-#endif
-            }
-#if DEBUG
-            PrintOpcodeStats();
-#endif
+            ExecuteWithGasBudget(Benchmark_Opcode.OneGasDatoshi);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void ExecuteUntilGas(long gasBudget)
+        {
+            ExecuteWithGasBudget(gasBudget);
         }
 
         protected override void OnFault(Exception ex)
@@ -197,6 +146,64 @@ private static JumpTable ComposeJumpTable()
             return jumpTable;
         }
 
+        private void ExecuteWithGasBudget(long gasBudget)
+        {
+            _gasConsumed = 0;
+            while (State != VMState.HALT && State != VMState.FAULT)
+            {
+                var instruction = CurrentContext!.CurrentInstruction ?? VM.Instruction.RET;
+                ExecuteStep(instruction);
+                if (State == VMState.HALT || State == VMState.FAULT)
+                    break;
+                if (ConsumeGas(instruction.OpCode, gasBudget))
+                    break;
+            }
+#if DEBUG
+            PrintOpcodeStats();
+#endif
+        }
+
+        private bool ConsumeGas(VM.OpCode opcode, long gasBudget)
+        {
+            if (gasBudget <= 0)
+                return true;
+            if (!Benchmark_Opcode.OpCodePrices.TryGetValue(opcode, out var price))
+                throw new KeyNotFoundException($"Missing benchmark gas price for opcode {opcode}.");
+            _gasConsumed += price;
+            if (_gasConsumed >= gasBudget)
+            {
+                State = VMState.HALT;
+                return true;
+            }
+            return false;
+        }
+
+        private void ExecuteStep(VM.Instruction instruction)
+        {
+            var allocatedBefore = GC.GetAllocatedBytesForCurrentThread();
+            Stopwatch? stopwatch = null;
+#if DEBUG
+            stopwatch = Stopwatch.StartNew();
+#else
+            if (Recorder is not null)
+                stopwatch = Stopwatch.StartNew();
+#endif
+            BeforeInstruction?.Invoke(this, instruction);
+            ExecuteNext();
+            var elapsed = stopwatch?.Elapsed ?? TimeSpan.Zero;
+            var allocatedAfter = GC.GetAllocatedBytesForCurrentThread();
+            var allocatedBytes = Math.Max(0, allocatedAfter - allocatedBefore);
+#if DEBUG
+            UpdateOpcodeStats(instruction.OpCode, elapsed);
+#endif
+            var stackDepth = CurrentContext?.EvaluationStack.Count ?? 0;
+            var altStackDepth = 0;
+            if (!Benchmark_Opcode.OpCodePrices.TryGetValue(instruction.OpCode, out var gas))
+                throw new KeyNotFoundException($"Missing benchmark gas price for opcode {instruction.OpCode}.");
+            Recorder?.RecordInstruction(instruction.OpCode, elapsed, allocatedBytes, stackDepth, altStackDepth, gas);
+            AfterInstruction?.Invoke(this, instruction);
+        }
+
         private static void OnSysCall(ExecutionEngine engine, VM.Instruction instruction)
         {
             uint method = instruction.TokenU32;

benchmarks/Neo.VM.Benchmarks/Program.cs

@@ -10,22 +10,68 @@
 // modifications are permitted.
 
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Running;
 using Neo.VM.Benchmark;
+using Neo.VM.Benchmark.Infrastructure;
+using Neo.VM.Benchmark.Native;
+using Neo.VM.Benchmark.OpCode;
+using Neo.VM.Benchmark.Syscalls;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
 using System.Reflection;
 
-// Define the benchmark or execute class
-if (Environment.GetEnvironmentVariable("NEO_VM_BENCHMARK") != null)
+var runnerArgs = FilterSpecialArguments(args, out var runPocs);
+var artifactsRoot = ResolveArtifactsRoot();
+EnsureBenchmarkEnvironment();
+
+if (runPocs)
+{
+    RunProofOfConcepts();
+    return;
+}
+
+RunBenchmarks(runnerArgs, artifactsRoot);
+
+static string[] FilterSpecialArguments(string[] args, out bool runPocs)
+{
+    var remaining = new List<string>(args.Length);
+    runPocs = false;
+
+    foreach (var arg in args)
+    {
+        if (string.Equals(arg, "--pocs", StringComparison.OrdinalIgnoreCase))
+        {
+            runPocs = true;
+            continue;
+        }
+
+        remaining.Add(arg);
+    }
+
+    return remaining.ToArray();
+}
+
+static string ResolveArtifactsRoot()
 {
-    BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
+    var root = Environment.GetEnvironmentVariable("NEO_BENCHMARK_ARTIFACTS")
+               ?? Path.Combine(AppContext.BaseDirectory, "BenchmarkArtifacts");
+    Directory.CreateDirectory(root);
+    Environment.SetEnvironmentVariable("NEO_BENCHMARK_ARTIFACTS", root);
+    return root;
 }
-else
+
+static void RunProofOfConcepts()
 {
     var benchmarkType = typeof(Benchmarks_PoCs);
-    var instance = Activator.CreateInstance(benchmarkType);
+    var instance = Activator.CreateInstance(benchmarkType)
+                   ?? throw new InvalidOperationException($"Unable to create instance of {benchmarkType.FullName}.");
+
     benchmarkType.GetMethods(BindingFlags.Public | BindingFlags.Instance)
         .FirstOrDefault(m => m.GetCustomAttribute<GlobalSetupAttribute>() != null)?
-        .Invoke(instance, null); // setup
+        .Invoke(instance, null);
 
     var methods = benchmarkType.GetMethods(BindingFlags.Public | BindingFlags.Instance)
         .Where(m => m.DeclaringType == benchmarkType && !m.GetCustomAttributes<GlobalSetupAttribute>().Any());
@@ -34,3 +80,57 @@
         method.Invoke(instance, null);
     }
 }
+
+static void RunBenchmarks(string[] benchmarkArgs, string artifactsRoot)
+{
+    BenchmarkArtifactRegistry.Reset();
+
+    var switcher = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly);
+    var config = ManualConfig
+        .Create(DefaultConfig.Instance)
+        .WithArtifactsPath(artifactsRoot)
+        .WithOptions(ConfigOptions.DisableOptimizationsValidator | ConfigOptions.KeepBenchmarkFiles);
+    if (benchmarkArgs.Length > 0)
+    {
+        switcher.Run(benchmarkArgs, config);
+    }
+    else
+    {
+        switcher.RunAllJoined(config);
+    }
+
+    BenchmarkArtifactRegistry.CollectFromDisk(artifactsRoot);
+
+    var missingOpcodes = OpcodeCoverageReport.GetUncoveredOpcodes();
+    var missingSyscalls = SyscallCoverageReport.GetMissing();
+    var missingNative = NativeCoverageReport.GetMissing();
+    if (BenchmarkArtifactRegistry.GetMetricArtifacts().Count == 0)
+    {
+        Console.WriteLine("[Benchmark] No metrics detected from BenchmarkDotNet run, executing manual pass...");
+        ManualSuiteRunner.RunAll();
+        BenchmarkArtifactRegistry.CollectFromDisk(artifactsRoot);
+    }
+
+    var report = BenchmarkFinalReportWriter.Write(artifactsRoot, missingOpcodes, missingSyscalls, missingNative);
+    BenchmarkFinalReportWriter.PrintToConsole(report);
+
+    var hasGaps = missingOpcodes.Count > 0 || missingSyscalls.Count > 0 || missingNative.Count > 0;
+    if (hasGaps)
+    {
+        Console.WriteLine("Benchmark run completed with missing coverage entries.");
+        Environment.ExitCode = 1;
+    }
+    else
+    {
+        Console.WriteLine("Benchmark run completed with full coverage.");
+        Environment.ExitCode = 0;
+    }
+}
+
+static void EnsureBenchmarkEnvironment()
+{
+    if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("NEO_VM_BENCHMARK")))
+        Environment.SetEnvironmentVariable("NEO_VM_BENCHMARK", "1");
+    if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("NEO_BENCHMARK_COVERAGE")))
+        Environment.SetEnvironmentVariable("NEO_BENCHMARK_COVERAGE", "1");
+}