diff --git a/.gitignore b/.gitignore index af82d040ba..2a0449522d 100644 --- a/.gitignore +++ b/.gitignore @@ -267,3 +267,4 @@ launchSettings.json # Benchmarks **/BenchmarkDotNet.Artifacts/ /src/Neo.CLI/neo-cli/ +.serena/ \ No newline at end of file diff --git a/benchmarks/Neo.VM.Benchmarks/Neo.VM.Benchmarks.csproj b/benchmarks/Neo.VM.Benchmarks/Neo.VM.Benchmarks.csproj index e0de03af9a..77305463e5 100644 --- a/benchmarks/Neo.VM.Benchmarks/Neo.VM.Benchmarks.csproj +++ b/benchmarks/Neo.VM.Benchmarks/Neo.VM.Benchmarks.csproj @@ -9,6 +9,7 @@ + diff --git a/benchmarks/Neo.VM.Benchmarks/OpCode/BenchmarkEngine.cs b/benchmarks/Neo.VM.Benchmarks/OpCode/BenchmarkEngine.cs index c913c4659f..b309cfe054 100644 --- a/benchmarks/Neo.VM.Benchmarks/OpCode/BenchmarkEngine.cs +++ b/benchmarks/Neo.VM.Benchmarks/OpCode/BenchmarkEngine.cs @@ -9,7 +9,9 @@ // Redistribution and use in source and binary forms with or without // modifications are permitted. +using Neo.VM.Benchmark.Infrastructure; using Neo.VM.Types; +using System; using System.Diagnostics; using System.Runtime.CompilerServices; @@ -22,7 +24,11 @@ public class BenchmarkEngine : ExecutionEngine { private readonly Dictionary _opcodeStats = new(); private readonly Dictionary> _breakPoints = new(); - private long _gasConsumed = 0; + private long _gasConsumed; + + public BenchmarkResultRecorder? Recorder { get; set; } + public Action? BeforeInstruction { get; set; } + public Action? AfterInstruction { get; set; } public BenchmarkEngine() : base(ComposeJumpTable()) { } @@ -70,14 +76,8 @@ public void ExecuteBenchmark() { while (State != VMState.HALT && State != VMState.FAULT) { -#if DEBUG - var stopwatch = Stopwatch.StartNew(); -#endif - ExecuteNext(); -#if DEBUG - stopwatch.Stop(); - UpdateOpcodeStats(CurrentContext!.CurrentInstruction!.OpCode, stopwatch.Elapsed); -#endif + var instruction = CurrentContext!.CurrentInstruction ?? VM.Instruction.RET; + ExecuteStep(instruction); } #if DEBUG PrintOpcodeStats(); @@ -87,76 +87,25 @@ public void ExecuteBenchmark() [MethodImpl(MethodImplOptions.AggressiveInlining)] public void ExecuteOneGASBenchmark() { - while (State != VMState.HALT && State != VMState.FAULT) - { - var instruction = CurrentContext!.CurrentInstruction ?? VM.Instruction.RET; - _gasConsumed += Benchmark_Opcode.OpCodePrices[instruction.OpCode]; - if (_gasConsumed >= Benchmark_Opcode.OneGasDatoshi) - { - State = VMState.HALT; - } -#if DEBUG - var stopwatch = Stopwatch.StartNew(); -#endif - ExecuteNext(); -#if DEBUG - stopwatch.Stop(); - UpdateOpcodeStats(instruction.OpCode, stopwatch.Elapsed); -#endif - } -#if DEBUG - PrintOpcodeStats(); -#endif + ExecuteWithGasBudget(Benchmark_Opcode.OneGasDatoshi); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public void ExecuteTwentyGASBenchmark() { - while (State != VMState.HALT && State != VMState.FAULT) - { - var instruction = CurrentContext!.CurrentInstruction ?? VM.Instruction.RET; - _gasConsumed += Benchmark_Opcode.OpCodePrices[instruction.OpCode]; - if (_gasConsumed >= 20 * Benchmark_Opcode.OneGasDatoshi) - { - State = VMState.HALT; - } -#if DEBUG - var stopwatch = Stopwatch.StartNew(); -#endif - ExecuteNext(); -#if DEBUG - stopwatch.Stop(); - UpdateOpcodeStats(instruction.OpCode, stopwatch.Elapsed); -#endif - } -#if DEBUG - PrintOpcodeStats(); -#endif + ExecuteWithGasBudget(20 * Benchmark_Opcode.OneGasDatoshi); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public void ExecuteOpCodesBenchmark() { - while (State != VMState.HALT && State != VMState.FAULT) - { - var instruction = CurrentContext!.CurrentInstruction ?? VM.Instruction.RET; - _gasConsumed += Benchmark_Opcode.OpCodePrices[instruction.OpCode]; - if (_gasConsumed >= Benchmark_Opcode.OneGasDatoshi) - { - State = VMState.HALT; - } -#if DEBUG - var stopwatch = Stopwatch.StartNew(); -#endif - ExecuteNext(); -#if DEBUG - stopwatch.Stop(); - UpdateOpcodeStats(instruction.OpCode, stopwatch.Elapsed); -#endif - } -#if DEBUG - PrintOpcodeStats(); -#endif + ExecuteWithGasBudget(Benchmark_Opcode.OneGasDatoshi); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void ExecuteUntilGas(long gasBudget) + { + ExecuteWithGasBudget(gasBudget); } protected override void OnFault(Exception ex) @@ -197,6 +146,64 @@ private static JumpTable ComposeJumpTable() return jumpTable; } + private void ExecuteWithGasBudget(long gasBudget) + { + _gasConsumed = 0; + while (State != VMState.HALT && State != VMState.FAULT) + { + var instruction = CurrentContext!.CurrentInstruction ?? VM.Instruction.RET; + ExecuteStep(instruction); + if (State == VMState.HALT || State == VMState.FAULT) + break; + if (ConsumeGas(instruction.OpCode, gasBudget)) + break; + } +#if DEBUG + PrintOpcodeStats(); +#endif + } + + private bool ConsumeGas(VM.OpCode opcode, long gasBudget) + { + if (gasBudget <= 0) + return true; + if (!Benchmark_Opcode.OpCodePrices.TryGetValue(opcode, out var price)) + throw new KeyNotFoundException($"Missing benchmark gas price for opcode {opcode}."); + _gasConsumed += price; + if (_gasConsumed >= gasBudget) + { + State = VMState.HALT; + return true; + } + return false; + } + + private void ExecuteStep(VM.Instruction instruction) + { + var allocatedBefore = GC.GetAllocatedBytesForCurrentThread(); + Stopwatch? stopwatch = null; +#if DEBUG + stopwatch = Stopwatch.StartNew(); +#else + if (Recorder is not null) + stopwatch = Stopwatch.StartNew(); +#endif + BeforeInstruction?.Invoke(this, instruction); + ExecuteNext(); + var elapsed = stopwatch?.Elapsed ?? TimeSpan.Zero; + var allocatedAfter = GC.GetAllocatedBytesForCurrentThread(); + var allocatedBytes = Math.Max(0, allocatedAfter - allocatedBefore); +#if DEBUG + UpdateOpcodeStats(instruction.OpCode, elapsed); +#endif + var stackDepth = CurrentContext?.EvaluationStack.Count ?? 0; + var altStackDepth = 0; + if (!Benchmark_Opcode.OpCodePrices.TryGetValue(instruction.OpCode, out var gas)) + throw new KeyNotFoundException($"Missing benchmark gas price for opcode {instruction.OpCode}."); + Recorder?.RecordInstruction(instruction.OpCode, elapsed, allocatedBytes, stackDepth, altStackDepth, gas); + AfterInstruction?.Invoke(this, instruction); + } + private static void OnSysCall(ExecutionEngine engine, VM.Instruction instruction) { uint method = instruction.TokenU32; diff --git a/benchmarks/Neo.VM.Benchmarks/Program.cs b/benchmarks/Neo.VM.Benchmarks/Program.cs index 75684f1349..2807ced0bf 100644 --- a/benchmarks/Neo.VM.Benchmarks/Program.cs +++ b/benchmarks/Neo.VM.Benchmarks/Program.cs @@ -10,22 +10,68 @@ // modifications are permitted. using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; using BenchmarkDotNet.Running; using Neo.VM.Benchmark; +using Neo.VM.Benchmark.Infrastructure; +using Neo.VM.Benchmark.Native; +using Neo.VM.Benchmark.OpCode; +using Neo.VM.Benchmark.Syscalls; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; using System.Reflection; -// Define the benchmark or execute class -if (Environment.GetEnvironmentVariable("NEO_VM_BENCHMARK") != null) +var runnerArgs = FilterSpecialArguments(args, out var runPocs); +var artifactsRoot = ResolveArtifactsRoot(); +EnsureBenchmarkEnvironment(); + +if (runPocs) +{ + RunProofOfConcepts(); + return; +} + +RunBenchmarks(runnerArgs, artifactsRoot); + +static string[] FilterSpecialArguments(string[] args, out bool runPocs) +{ + var remaining = new List(args.Length); + runPocs = false; + + foreach (var arg in args) + { + if (string.Equals(arg, "--pocs", StringComparison.OrdinalIgnoreCase)) + { + runPocs = true; + continue; + } + + remaining.Add(arg); + } + + return remaining.ToArray(); +} + +static string ResolveArtifactsRoot() { - BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); + var root = Environment.GetEnvironmentVariable("NEO_BENCHMARK_ARTIFACTS") + ?? Path.Combine(AppContext.BaseDirectory, "BenchmarkArtifacts"); + Directory.CreateDirectory(root); + Environment.SetEnvironmentVariable("NEO_BENCHMARK_ARTIFACTS", root); + return root; } -else + +static void RunProofOfConcepts() { var benchmarkType = typeof(Benchmarks_PoCs); - var instance = Activator.CreateInstance(benchmarkType); + var instance = Activator.CreateInstance(benchmarkType) + ?? throw new InvalidOperationException($"Unable to create instance of {benchmarkType.FullName}."); + benchmarkType.GetMethods(BindingFlags.Public | BindingFlags.Instance) .FirstOrDefault(m => m.GetCustomAttribute() != null)? - .Invoke(instance, null); // setup + .Invoke(instance, null); var methods = benchmarkType.GetMethods(BindingFlags.Public | BindingFlags.Instance) .Where(m => m.DeclaringType == benchmarkType && !m.GetCustomAttributes().Any()); @@ -34,3 +80,57 @@ method.Invoke(instance, null); } } + +static void RunBenchmarks(string[] benchmarkArgs, string artifactsRoot) +{ + BenchmarkArtifactRegistry.Reset(); + + var switcher = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly); + var config = ManualConfig + .Create(DefaultConfig.Instance) + .WithArtifactsPath(artifactsRoot) + .WithOptions(ConfigOptions.DisableOptimizationsValidator | ConfigOptions.KeepBenchmarkFiles); + if (benchmarkArgs.Length > 0) + { + switcher.Run(benchmarkArgs, config); + } + else + { + switcher.RunAllJoined(config); + } + + BenchmarkArtifactRegistry.CollectFromDisk(artifactsRoot); + + var missingOpcodes = OpcodeCoverageReport.GetUncoveredOpcodes(); + var missingSyscalls = SyscallCoverageReport.GetMissing(); + var missingNative = NativeCoverageReport.GetMissing(); + if (BenchmarkArtifactRegistry.GetMetricArtifacts().Count == 0) + { + Console.WriteLine("[Benchmark] No metrics detected from BenchmarkDotNet run, executing manual pass..."); + ManualSuiteRunner.RunAll(); + BenchmarkArtifactRegistry.CollectFromDisk(artifactsRoot); + } + + var report = BenchmarkFinalReportWriter.Write(artifactsRoot, missingOpcodes, missingSyscalls, missingNative); + BenchmarkFinalReportWriter.PrintToConsole(report); + + var hasGaps = missingOpcodes.Count > 0 || missingSyscalls.Count > 0 || missingNative.Count > 0; + if (hasGaps) + { + Console.WriteLine("Benchmark run completed with missing coverage entries."); + Environment.ExitCode = 1; + } + else + { + Console.WriteLine("Benchmark run completed with full coverage."); + Environment.ExitCode = 0; + } +} + +static void EnsureBenchmarkEnvironment() +{ + if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("NEO_VM_BENCHMARK"))) + Environment.SetEnvironmentVariable("NEO_VM_BENCHMARK", "1"); + if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("NEO_BENCHMARK_COVERAGE"))) + Environment.SetEnvironmentVariable("NEO_BENCHMARK_COVERAGE", "1"); +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/ApplicationEngineVmScenario.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/ApplicationEngineVmScenario.cs new file mode 100644 index 0000000000..81a6e6111e --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/ApplicationEngineVmScenario.cs @@ -0,0 +1,76 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// ApplicationEngineVmScenario.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo.SmartContract; +using System; + +namespace Neo.VM.Benchmark.Infrastructure +{ + /// + /// Scenario wrapper executing scripts under . + /// + internal sealed class ApplicationEngineVmScenario : IVmScenario + { + private readonly Func _scriptFactory; + private readonly Action? _configure; + private readonly Func? _engineFactory; + + public ApplicationEngineVmScenario( + Func scriptFactory, + Action? configure = null, + Func? engineFactory = null) + { + _scriptFactory = scriptFactory; + _configure = configure; + _engineFactory = engineFactory; + } + + public void Execute(BenchmarkVariant variant) + { + var baseProfile = BenchmarkExecutionContext.CurrentCase?.Profile + ?? BenchmarkExecutionContext.CurrentProfile + ?? ScenarioProfile.For(ScenarioComplexity.Standard); + + var scripts = _scriptFactory(baseProfile); + var script = scripts.GetScript(variant); + var effectiveProfile = script.Profile.IsEmpty ? baseProfile : script.Profile; + + BenchmarkExecutionContext.SetProfileOverride(effectiveProfile); + Execute(script); + } + + private void Execute(ApplicationEngineScript script) + { + var profile = BenchmarkExecutionContext.CurrentProfile ?? ScenarioProfile.For(ScenarioComplexity.Standard); + using var engine = _engineFactory?.Invoke(profile) ?? BenchmarkApplicationEngine.Create(); + engine.Recorder = BenchmarkExecutionContext.CurrentRecorder; + _configure?.Invoke(engine, profile); + engine.LoadScript(script.Script, configureState: state => script.ConfigureState?.Invoke(state)); + engine.Execute(); + } + + public readonly record struct ApplicationEngineScript(byte[] Script, ScenarioProfile Profile, Action? ConfigureState = null) + { + public static implicit operator ApplicationEngineScript(byte[] script) => new(script, default); + } + + public readonly record struct ApplicationEngineScriptSet(ApplicationEngineScript Baseline, ApplicationEngineScript Single, ApplicationEngineScript Saturated) + { + public ApplicationEngineScript GetScript(BenchmarkVariant variant) => variant switch + { + BenchmarkVariant.Baseline => Baseline, + BenchmarkVariant.Single => Single, + BenchmarkVariant.Saturated => Saturated, + _ => throw new ArgumentOutOfRangeException(nameof(variant), variant, null) + }; + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkApplicationEngine.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkApplicationEngine.cs new file mode 100644 index 0000000000..103e2ab12e --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkApplicationEngine.cs @@ -0,0 +1,178 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// BenchmarkApplicationEngine.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo; +using Neo.Network.P2P.Payloads; +using Neo.Persistence; +using Neo.Persistence.Providers; +using Neo.SmartContract; +using Neo.SmartContract.Native; +using Neo.VM; +using System; +using System.Collections; +using System.Diagnostics; +using System.Reflection; + +namespace Neo.VM.Benchmark.Infrastructure +{ + /// + /// Thin wrapper around that records syscall and native-method timings. + /// + internal sealed class BenchmarkApplicationEngine : ApplicationEngine + { + private static readonly MethodInfo? s_getContractMethods = typeof(NativeContract) + .GetMethod("GetContractMethods", BindingFlags.Instance | BindingFlags.NonPublic); + + private readonly StoreCache _snapshot; + + private BenchmarkApplicationEngine( + TriggerType trigger, + StoreCache snapshot, + IVerifiable? container, + Block persistingBlock, + ProtocolSettings settings, + long gas, + IDiagnostic? diagnostic, + JumpTable jumpTable) + : base(trigger, container, snapshot, persistingBlock, settings, gas, diagnostic, jumpTable) + { + _snapshot = snapshot; + } + + public BenchmarkResultRecorder? Recorder { get; set; } + + public static BenchmarkApplicationEngine Create( + ProtocolSettings? settings = null, + long? gas = null, + IVerifiable? container = null, + StoreCache? snapshot = null, + TriggerType trigger = TriggerType.Application, + Block? persistingBlock = null, + IDiagnostic? diagnostic = null) + { + settings = BenchmarkProtocolSettings.ResolveSettings(settings); + snapshot ??= NativeBenchmarkStateFactory.CreateSnapshot(); + persistingBlock ??= CreateBenchmarkBlock(); + var index = persistingBlock.Index; + var jumpTable = settings.IsHardforkEnabled(Hardfork.HF_Echidna, index) ? DefaultJumpTable : NotEchidnaJumpTable; + return new BenchmarkApplicationEngine(trigger, snapshot, container, persistingBlock, settings, gas ?? TestModeGas, diagnostic, jumpTable); + } + + protected override void OnSysCall(InteropDescriptor descriptor) + { + if (Recorder is null) + { + base.OnSysCall(descriptor); + return; + } + + var allocatedBefore = GC.GetAllocatedBytesForCurrentThread(); + var gasBefore = FeeConsumed; + var stopwatch = Stopwatch.StartNew(); + string? nativeMethodName = null; + if (descriptor == System_Contract_CallNative) + { + nativeMethodName = TryResolveNativeCall(); + } + + try + { + base.OnSysCall(descriptor); + } + finally + { + stopwatch.Stop(); + var allocatedAfter = GC.GetAllocatedBytesForCurrentThread(); + var allocatedBytes = Math.Max(0, allocatedAfter - allocatedBefore); + var gasAfter = FeeConsumed; + var gasDelta = Math.Max(0, gasAfter - gasBefore); + var stackDepth = CurrentContext?.EvaluationStack.Count ?? 0; + var altStackDepth = 0; + if (nativeMethodName is not null) + { + Recorder.RecordNativeMethod(nativeMethodName, stopwatch.Elapsed, allocatedBytes, stackDepth, altStackDepth, gasDelta); + } + else + { + Recorder.RecordSyscall(descriptor.Name, stopwatch.Elapsed, allocatedBytes, stackDepth, altStackDepth, gasDelta); + } + } + } + + private string? TryResolveNativeCall() + { + var context = CurrentContext; + if (context is null) + return null; + + var contract = NativeContract.GetContract(CurrentScriptHash); + if (contract is null) + return null; + + if (s_getContractMethods is null) + return contract.Name; + + if (s_getContractMethods.Invoke(contract, new object[] { this }) is not IDictionary methods) + return contract.Name; + + var pointer = context.InstructionPointer; + object? metadata = null; + + if (methods is IDictionary dict && dict.Contains(pointer)) + { + metadata = dict[pointer]; + } + else + { + foreach (DictionaryEntry entry in methods) + { + if (entry.Key is int key && key == pointer) + { + metadata = entry.Value; + break; + } + } + } + + var name = metadata is null + ? "unknown" + : metadata.GetType().GetProperty("Name", BindingFlags.Public | BindingFlags.Instance)?.GetValue(metadata) as string ?? "unknown"; + + return string.IsNullOrEmpty(name) + ? contract.Name + : $"{contract.Name}.{name}"; + } + + public override void Dispose() + { + base.Dispose(); + _snapshot.Dispose(); + } + + private static Block CreateBenchmarkBlock() + { + return new Block + { + Header = new Header + { + Version = 0, + PrevHash = UInt256.Zero, + MerkleRoot = UInt256.Zero, + Timestamp = (ulong)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), + Index = 1, + NextConsensus = UInt160.Zero, + Witness = Witness.Empty + }, + Transactions = Array.Empty() + }; + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkArtifactRegistry.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkArtifactRegistry.cs new file mode 100644 index 0000000000..d09ca8696d --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkArtifactRegistry.cs @@ -0,0 +1,122 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// BenchmarkArtifactRegistry.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Neo.VM.Benchmark.Infrastructure +{ + /// + /// Tracks benchmark artifact files so we can produce a consolidated report once all suites finish. + /// + internal static class BenchmarkArtifactRegistry + { + private static readonly object s_sync = new(); + private static readonly List<(BenchmarkComponent Component, string Path)> s_metricArtifacts = new(); + private static readonly List<(string Category, string Path)> s_coverageArtifacts = new(); + + public static void Reset() + { + lock (s_sync) + { + s_metricArtifacts.Clear(); + s_coverageArtifacts.Clear(); + } + } + + public static void RegisterMetrics(BenchmarkComponent component, string path) + { + if (string.IsNullOrEmpty(path)) + return; + + lock (s_sync) + { + if (!s_metricArtifacts.Any(entry => entry.Path == path)) + s_metricArtifacts.Add((component, path)); + } + } + + public static void RegisterCoverage(string category, string path) + { + if (string.IsNullOrEmpty(category) || string.IsNullOrEmpty(path)) + return; + + lock (s_sync) + { + if (!s_coverageArtifacts.Any(entry => entry.Path == path)) + s_coverageArtifacts.Add((category, path)); + } + } + + public static IReadOnlyList<(BenchmarkComponent Component, string Path)> GetMetricArtifacts() + { + lock (s_sync) + { + return s_metricArtifacts.OrderBy(static entry => entry.Component).ThenBy(static entry => entry.Path).ToArray(); + } + } + + public static IReadOnlyList<(string Category, string Path)> GetCoverageArtifacts() + { + lock (s_sync) + { + return s_coverageArtifacts.OrderBy(static entry => entry.Category).ThenBy(static entry => entry.Path).ToArray(); + } + } + + public static void CollectFromDisk(string root) + { + if (string.IsNullOrEmpty(root) || !Directory.Exists(root)) + return; + + CollectDirectory(root); + + var parent = Path.GetDirectoryName(root); + if (string.IsNullOrEmpty(parent) || !Directory.Exists(parent)) + return; + + foreach (var jobDir in Directory.EnumerateDirectories(parent, "Neo.VM.Benchmarks-*", SearchOption.TopDirectoryOnly)) + { + var jobArtifacts = Path.Combine(jobDir, "BenchmarkArtifacts"); + if (Directory.Exists(jobArtifacts)) + CollectDirectory(jobArtifacts); + } + } + + private static void CollectDirectory(string directory) + { + foreach (var file in Directory.EnumerateFiles(directory, "*-metrics-*.csv", SearchOption.AllDirectories)) + { + RegisterMetrics(InferComponent(file), file); + } + + foreach (var file in Directory.EnumerateFiles(directory, "*-missing.csv", SearchOption.AllDirectories)) + { + var category = Path.GetFileNameWithoutExtension(file); + RegisterCoverage(category ?? string.Empty, file); + } + } + + private static BenchmarkComponent InferComponent(string path) + { + var fileName = Path.GetFileName(path) ?? string.Empty; + if (fileName.Contains("opcode", StringComparison.OrdinalIgnoreCase)) + return BenchmarkComponent.Opcode; + if (fileName.Contains("syscall", StringComparison.OrdinalIgnoreCase)) + return BenchmarkComponent.Syscall; + if (fileName.Contains("native", StringComparison.OrdinalIgnoreCase)) + return BenchmarkComponent.NativeContract; + return BenchmarkComponent.Opcode; + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkComponents.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkComponents.cs new file mode 100644 index 0000000000..451bce2e4b --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkComponents.cs @@ -0,0 +1,43 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// BenchmarkComponents.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +namespace Neo.VM.Benchmark.Infrastructure +{ + /// + /// Identifies the high level component a scenario exercises. + /// + public enum BenchmarkComponent + { + Opcode, + Syscall, + NativeContract + } + + /// + /// Identifies the BenchmarkDotNet variant a scenario may execute. + /// + public enum BenchmarkVariant + { + Baseline, + Single, + Saturated + } + + /// + /// Identifies the kind of operation being measured. + /// + public enum BenchmarkOperationKind + { + Instruction, + Syscall, + NativeMethod + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkDataFactory.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkDataFactory.cs new file mode 100644 index 0000000000..8f54b7500b --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkDataFactory.cs @@ -0,0 +1,70 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// BenchmarkDataFactory.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo.VM; +using System; +using System.Collections.Generic; + +namespace Neo.VM.Benchmark.Infrastructure +{ + internal static class BenchmarkDataFactory + { + private static readonly ExecutionEngineLimits Limits = ExecutionEngineLimits.Default; + private static int ClampLength(int length) + { + return Math.Clamp(length, 1, (int)Limits.MaxItemSize); + } + + public static byte[] CreateByteArray(int length, byte fill = 0x42) + { + length = ClampLength(length); + var buffer = new byte[length]; + for (int i = 0; i < buffer.Length; i++) + buffer[i] = unchecked((byte)(fill + i)); + return buffer; + } + + public static IReadOnlyList CreateByteSegments(int count, int segmentLength, byte seed = 0x10) + { + count = Math.Max(1, count); + segmentLength = ClampLength(segmentLength); + var result = new byte[count][]; + for (int i = 0; i < count; i++) + { + result[i] = CreateByteArray(segmentLength, (byte)(seed + i)); + } + return result; + } + + public static byte[] CreateIteratorKey(int index, byte prefix = 0xC1) + { + return new[] { prefix, unchecked((byte)index) }; + } + + public static string CreateString(int length, char seed = 'a') + { + length = ClampLength(length); + var buffer = new char[length]; + for (int i = 0; i < length; i++) + buffer[i] = (char)(seed + (i % 26)); + return new string(buffer); + } + + public static string CreateNumericString(int length) + { + length = ClampLength(length); + var buffer = new char[length]; + for (int i = 0; i < length; i++) + buffer[i] = (char)('0' + (i % 10)); + return new string(buffer); + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkExecutionContext.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkExecutionContext.cs new file mode 100644 index 0000000000..eec32509e7 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkExecutionContext.cs @@ -0,0 +1,60 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// BenchmarkExecutionContext.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +namespace Neo.VM.Benchmark.Infrastructure +{ + /// + /// Holds ambient state for benchmark execution (per-thread). + /// + public static class BenchmarkExecutionContext + { + private static readonly System.Threading.AsyncLocal s_recorder = new(); + private static readonly System.Threading.AsyncLocal s_case = new(); + private static readonly System.Threading.AsyncLocal s_variant = new(); + private static readonly System.Threading.AsyncLocal s_profileOverride = new(); + + public static BenchmarkResultRecorder? CurrentRecorder + { + get => s_recorder.Value; + set => s_recorder.Value = value; + } + + public static VmBenchmarkCase? CurrentCase + { + get => s_case.Value; + set => s_case.Value = value; + } + + public static BenchmarkVariant? CurrentVariant + { + get => s_variant.Value; + set => s_variant.Value = value; + } + + public static ScenarioProfile? CurrentProfile => s_profileOverride.Value ?? CurrentCase?.Profile; + + public static ScenarioComplexity? CurrentComplexity => CurrentCase?.Complexity; + + public static BenchmarkComponent? CurrentComponent => CurrentCase?.Component; + + public static void SetProfileOverride(ScenarioProfile profile) + { + s_profileOverride.Value = profile; + } + + public static void ClearVariant() + { + s_variant.Value = null; + s_case.Value = null; + s_profileOverride.Value = null; + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkExecutionSummary.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkExecutionSummary.cs new file mode 100644 index 0000000000..1a66feb065 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkExecutionSummary.cs @@ -0,0 +1,78 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// BenchmarkExecutionSummary.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using System.Globalization; +using System.IO; +using System.Linq; + +namespace Neo.VM.Benchmark.Infrastructure +{ + /// + /// Handles post-run persistence of recorded benchmark statistics. + /// + internal sealed class BenchmarkExecutionSummary + { + private readonly BenchmarkResultRecorder _recorder; + private readonly string _artifactPath; + + public BenchmarkExecutionSummary(BenchmarkResultRecorder recorder, string artifactPath) + { + _recorder = recorder; + _artifactPath = artifactPath; + } + + public void Write() + { + Directory.CreateDirectory(Path.GetDirectoryName(_artifactPath)!); + using var writer = new StreamWriter(_artifactPath, append: false); + writer.WriteLine("Component,ScenarioId,OperationKind,OperationId,Complexity,Variant,Iterations,DataLength,CollectionLength,TotalIterations,TotalDataBytes,TotalElements,Count,TotalMicroseconds,AverageNanoseconds,NanosecondsPerIteration,NanosecondsPerByte,NanosecondsPerElement,TotalAllocatedBytes,AllocatedBytesPerIteration,AllocatedBytesPerByte,AllocatedBytesPerElement,AverageStackDepth,PeakStackDepth,AverageAltStackDepth,PeakAltStackDepth,TotalGasConsumed,GasPerIteration"); + foreach (var measurement in _recorder + .Snapshot() + .OrderBy(static m => m.Component) + .ThenBy(static m => m.OperationKind) + .ThenBy(static m => m.OperationId) + .ThenBy(static m => m.Complexity) + .ThenBy(static m => m.Variant)) + { + writer.WriteLine(string.Join(',', + measurement.Component, + measurement.ScenarioId, + measurement.OperationKind, + measurement.OperationId, + measurement.Complexity, + measurement.Variant, + measurement.Profile.Iterations.ToString(CultureInfo.InvariantCulture), + measurement.Profile.DataLength.ToString(CultureInfo.InvariantCulture), + measurement.Profile.CollectionLength.ToString(CultureInfo.InvariantCulture), + measurement.TotalIterations.ToString(CultureInfo.InvariantCulture), + measurement.TotalDataBytes.ToString(CultureInfo.InvariantCulture), + measurement.TotalElements.ToString(CultureInfo.InvariantCulture), + measurement.Count.ToString(CultureInfo.InvariantCulture), + measurement.TotalMicroseconds.ToString("F2", CultureInfo.InvariantCulture), + measurement.AverageNanoseconds.ToString("F2", CultureInfo.InvariantCulture), + measurement.NanosecondsPerIteration.ToString("F2", CultureInfo.InvariantCulture), + measurement.NanosecondsPerByte.ToString("F4", CultureInfo.InvariantCulture), + measurement.NanosecondsPerElement.ToString("F4", CultureInfo.InvariantCulture), + measurement.TotalAllocatedBytes.ToString(CultureInfo.InvariantCulture), + measurement.AllocatedBytesPerIteration.ToString("F2", CultureInfo.InvariantCulture), + measurement.AllocatedBytesPerByte.ToString("F6", CultureInfo.InvariantCulture), + measurement.AllocatedBytesPerElement.ToString("F6", CultureInfo.InvariantCulture), + measurement.AverageStackDepth.ToString("F2", CultureInfo.InvariantCulture), + measurement.PeakStackDepth.ToString(CultureInfo.InvariantCulture), + measurement.AverageAltStackDepth.ToString("F2", CultureInfo.InvariantCulture), + measurement.PeakAltStackDepth.ToString(CultureInfo.InvariantCulture), + measurement.TotalGasConsumed.ToString(CultureInfo.InvariantCulture), + measurement.GasPerIteration.ToString("F4", CultureInfo.InvariantCulture))); + + } + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkFinalReportWriter.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkFinalReportWriter.cs new file mode 100644 index 0000000000..f13fb9f63e --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkFinalReportWriter.cs @@ -0,0 +1,365 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// BenchmarkFinalReportWriter.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo.VM.Benchmark.Native; +using Neo.VM.Benchmark.OpCode; +using Neo.VM.Benchmark.Syscalls; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using VmOpCode = Neo.VM.OpCode; + +namespace Neo.VM.Benchmark.Infrastructure +{ + internal static class BenchmarkFinalReportWriter + { + internal sealed record ReportSummary( + string SummaryPath, + string? CombinedMetricsPath, + IReadOnlyCollection<(BenchmarkComponent Component, string Path)> MetricSources, + IReadOnlyCollection<(string Category, string Path)> CoverageSources, + IReadOnlyCollection MissingOpcodes, + IReadOnlyCollection MissingSyscalls, + IReadOnlyCollection MissingNative, + string OpcodeMissingPath, + string InteropMissingPath, + string? ScalingPath); + + public static ReportSummary Write(string artifactsRoot, + IReadOnlyCollection missingOpcodes, + IReadOnlyCollection missingSyscalls, + IReadOnlyCollection missingNative) + { + Directory.CreateDirectory(artifactsRoot); + + var timestamp = DateTime.UtcNow.ToString("yyyyMMddHHmmss", CultureInfo.InvariantCulture); + + var metricSources = BenchmarkArtifactRegistry.GetMetricArtifacts(); + var coverageSources = BenchmarkArtifactRegistry.GetCoverageArtifacts(); + var combinedMetricsPath = MergeMetrics(artifactsRoot, timestamp, metricSources); + + var opcodeCoveragePath = Path.Combine(artifactsRoot, "opcode-coverage.csv"); + OpcodeCoverageReport.WriteCoverageTable(opcodeCoveragePath); + BenchmarkArtifactRegistry.RegisterCoverage("opcode-coverage", opcodeCoveragePath); + coverageSources = BenchmarkArtifactRegistry.GetCoverageArtifacts(); + + var opcodeMissingPath = Path.Combine(artifactsRoot, "opcode-missing.csv"); + OpcodeCoverageReport.WriteMissingList(opcodeMissingPath, missingOpcodes); + BenchmarkArtifactRegistry.RegisterCoverage("opcode-missing", opcodeMissingPath); + coverageSources = BenchmarkArtifactRegistry.GetCoverageArtifacts(); + + var interopMissingPath = Path.Combine(artifactsRoot, "interop-missing.csv"); + InteropCoverageReport.WriteReport(interopMissingPath, missingSyscalls, missingNative); + BenchmarkArtifactRegistry.RegisterCoverage("interop-missing", interopMissingPath); + coverageSources = BenchmarkArtifactRegistry.GetCoverageArtifacts(); + + var scalingPath = WriteScalingReport(artifactsRoot, combinedMetricsPath); + + var summaryPath = Path.Combine(artifactsRoot, $"benchmark-summary-{timestamp}.txt"); + WriteSummary(summaryPath, combinedMetricsPath, scalingPath, metricSources, coverageSources, missingOpcodes, missingSyscalls, missingNative, opcodeMissingPath, interopMissingPath); + + return new ReportSummary(summaryPath, combinedMetricsPath, metricSources, coverageSources, + missingOpcodes, missingSyscalls, missingNative, opcodeMissingPath, interopMissingPath, scalingPath); + } + + public static void PrintToConsole(ReportSummary summary) + { + Console.WriteLine(); + Console.WriteLine("================ Benchmark Summary ================"); + Console.WriteLine(summary.SummaryPath); + Console.WriteLine(); + + if (summary.MetricSources.Count > 0) + { + Console.WriteLine("Metrics"); + foreach (var (component, path) in summary.MetricSources) + Console.WriteLine($"- {component,-14} {path}"); + if (!string.IsNullOrEmpty(summary.CombinedMetricsPath)) + Console.WriteLine($"- Combined {summary.CombinedMetricsPath}"); + if (!string.IsNullOrEmpty(summary.ScalingPath)) + Console.WriteLine($"- Scaling {summary.ScalingPath}"); + Console.WriteLine(); + } + else + { + Console.WriteLine("Metrics"); + Console.WriteLine("- No metric artifacts were produced."); + Console.WriteLine(); + } + + Console.WriteLine("Coverage"); + Console.WriteLine($"- Missing opcodes : {summary.MissingOpcodes.Count}"); + Console.WriteLine($" Details : {summary.OpcodeMissingPath}"); + Console.WriteLine($"- Missing syscalls: {summary.MissingSyscalls.Count}"); + Console.WriteLine($"- Missing natives : {summary.MissingNative.Count}"); + Console.WriteLine($" Interop summary : {summary.InteropMissingPath}"); + foreach (var (category, path) in summary.CoverageSources) + { + Console.WriteLine($" {category,-17} {path}"); + } + Console.WriteLine("==================================================="); + Console.WriteLine(); + } + + private static string? MergeMetrics(string root, string timestamp, IReadOnlyCollection<(BenchmarkComponent Component, string Path)> metricSources) + { + if (metricSources.Count == 0) + return null; + + var combinedPath = Path.Combine(root, $"benchmark-metrics-{timestamp}.csv"); + bool headerWritten = false; + + using var writer = new StreamWriter(combinedPath, append: false); + foreach (var (_, path) in metricSources) + { + if (!File.Exists(path)) + continue; + + using var reader = new StreamReader(path); + string? header = reader.ReadLine(); + if (header is null) + continue; + + if (!headerWritten) + { + writer.WriteLine(header); + headerWritten = true; + } + + string? line; + while ((line = reader.ReadLine()) is not null) + { + writer.WriteLine(line); + } + } + + if (!headerWritten) + { + writer.Close(); + File.Delete(combinedPath); + return null; + } + + return combinedPath; + } + + private static void WriteSummary( + string summaryPath, + string? combinedMetricsPath, + string? scalingPath, + IReadOnlyCollection<(BenchmarkComponent Component, string Path)> metricSources, + IReadOnlyCollection<(string Category, string Path)> coverageSources, + IReadOnlyCollection missingOpcodes, + IReadOnlyCollection missingSyscalls, + IReadOnlyCollection missingNative, + string opcodeMissingPath, + string interopMissingPath) + { + using var writer = new StreamWriter(summaryPath, append: false); + writer.WriteLine($"NEO Benchmark Summary ({DateTime.UtcNow:O})"); + writer.WriteLine(); + + writer.WriteLine("Metrics"); + if (metricSources.Count == 0) + { + writer.WriteLine("- No metric artifacts were produced."); + } + else + { + foreach (var (component, path) in metricSources) + writer.WriteLine($"- {component,-14} {path}"); + if (!string.IsNullOrEmpty(combinedMetricsPath)) + writer.WriteLine($"- Combined {combinedMetricsPath}"); + if (!string.IsNullOrEmpty(scalingPath)) + writer.WriteLine($"- Scaling {scalingPath}"); + } + + writer.WriteLine(); + writer.WriteLine("Coverage"); + writer.WriteLine($"- Missing opcodes : {missingOpcodes.Count}"); + writer.WriteLine($" Details : {opcodeMissingPath}"); + writer.WriteLine($"- Missing syscalls: {missingSyscalls.Count}"); + writer.WriteLine($"- Missing natives : {missingNative.Count}"); + writer.WriteLine($" Interop summary : {interopMissingPath}"); + foreach (var (category, path) in coverageSources) + writer.WriteLine($"- {category,-17} {path}"); + } + + private static string? WriteScalingReport(string artifactsRoot, string? combinedMetricsPath) + { + if (string.IsNullOrEmpty(combinedMetricsPath) || !File.Exists(combinedMetricsPath)) + return null; + + var measurements = new List(); + using (var reader = new StreamReader(combinedMetricsPath)) + { + var header = reader.ReadLine(); + if (header is null) + return null; + + string? line; + while ((line = reader.ReadLine()) is not null) + { + if (ScalingMeasurement.TryParse(line, out var measurement)) + measurements.Add(measurement); + } + } + + if (measurements.Count == 0) + return null; + + var scalingPath = Path.Combine(artifactsRoot, "benchmark-scaling.csv"); + using var writer = new StreamWriter(scalingPath, append: false); + writer.WriteLine("Component,ScenarioId,OperationKind,OperationId,Complexity,BaselineNsPerIteration,SingleNsPerIteration,SaturatedNsPerIteration,IterationScale,BaselineNsPerByte,SingleNsPerByte,SaturatedNsPerByte,ByteScale,BaselineAllocatedPerIteration,SingleAllocatedPerIteration,SaturatedAllocatedPerIteration,AllocatedIterationScale,BaselineAllocatedPerByte,SingleAllocatedPerByte,SaturatedAllocatedPerByte,AllocatedByteScale,BaselineAvgStackDepth,SingleAvgStackDepth,SaturatedAvgStackDepth,StackDepthScale,BaselineAvgAltStackDepth,SingleAvgAltStackDepth,SaturatedAvgAltStackDepth,AltStackDepthScale,BaselineGasPerIteration,SingleGasPerIteration,SaturatedGasPerIteration,GasIterationScale,BaselineTotalDataBytes,SingleTotalDataBytes,SaturatedTotalDataBytes,BaselineTotalAllocatedBytes,SingleTotalAllocatedBytes,SaturatedTotalAllocatedBytes,BaselineTotalGas,SingleTotalGas,SaturatedTotalGas"); + + foreach (var group in measurements.GroupBy(m => (m.Component, m.ScenarioId, m.OperationKind, m.OperationId, m.Complexity))) + { + if (!group.Any()) + continue; + + var baseline = group.FirstOrDefault(m => m.Variant == BenchmarkVariant.Baseline) ?? group.First(); + var single = group.FirstOrDefault(m => m.Variant == BenchmarkVariant.Single) ?? baseline; + var saturated = group.FirstOrDefault(m => m.Variant == BenchmarkVariant.Saturated) ?? single; + + var iterationScale = single.NanosecondsPerIteration > 0 && saturated.NanosecondsPerIteration > 0 + ? saturated.NanosecondsPerIteration / single.NanosecondsPerIteration + : 0; + var byteScale = single.NanosecondsPerByte > 0 && saturated.NanosecondsPerByte > 0 + ? saturated.NanosecondsPerByte / single.NanosecondsPerByte + : 0; + var allocatedIterationScale = single.AllocatedBytesPerIteration > 0 && saturated.AllocatedBytesPerIteration > 0 + ? saturated.AllocatedBytesPerIteration / single.AllocatedBytesPerIteration + : 0; + var allocatedByteScale = single.AllocatedBytesPerByte > 0 && saturated.AllocatedBytesPerByte > 0 + ? saturated.AllocatedBytesPerByte / single.AllocatedBytesPerByte + : 0; + var stackDepthScale = single.AverageStackDepth > 0 && saturated.AverageStackDepth > 0 + ? saturated.AverageStackDepth / single.AverageStackDepth + : 0; + var altStackDepthScale = single.AverageAltStackDepth > 0 && saturated.AverageAltStackDepth > 0 + ? saturated.AverageAltStackDepth / single.AverageAltStackDepth + : 0; + var gasIterationScale = single.GasPerIteration > 0 && saturated.GasPerIteration > 0 + ? saturated.GasPerIteration / single.GasPerIteration + : 0; + + writer.WriteLine(string.Join(',', + group.Key.Component, + group.Key.ScenarioId, + group.Key.OperationKind, + group.Key.OperationId, + group.Key.Complexity, + baseline.NanosecondsPerIteration.ToString("F2", CultureInfo.InvariantCulture), + single.NanosecondsPerIteration.ToString("F2", CultureInfo.InvariantCulture), + saturated.NanosecondsPerIteration.ToString("F2", CultureInfo.InvariantCulture), + iterationScale.ToString("F4", CultureInfo.InvariantCulture), + baseline.NanosecondsPerByte.ToString("F4", CultureInfo.InvariantCulture), + single.NanosecondsPerByte.ToString("F4", CultureInfo.InvariantCulture), + saturated.NanosecondsPerByte.ToString("F4", CultureInfo.InvariantCulture), + byteScale.ToString("F4", CultureInfo.InvariantCulture), + baseline.AllocatedBytesPerIteration.ToString("F2", CultureInfo.InvariantCulture), + single.AllocatedBytesPerIteration.ToString("F2", CultureInfo.InvariantCulture), + saturated.AllocatedBytesPerIteration.ToString("F2", CultureInfo.InvariantCulture), + allocatedIterationScale.ToString("F4", CultureInfo.InvariantCulture), + baseline.AllocatedBytesPerByte.ToString("F6", CultureInfo.InvariantCulture), + single.AllocatedBytesPerByte.ToString("F6", CultureInfo.InvariantCulture), + saturated.AllocatedBytesPerByte.ToString("F6", CultureInfo.InvariantCulture), + allocatedByteScale.ToString("F4", CultureInfo.InvariantCulture), + baseline.AverageStackDepth.ToString("F2", CultureInfo.InvariantCulture), + single.AverageStackDepth.ToString("F2", CultureInfo.InvariantCulture), + saturated.AverageStackDepth.ToString("F2", CultureInfo.InvariantCulture), + stackDepthScale.ToString("F4", CultureInfo.InvariantCulture), + baseline.AverageAltStackDepth.ToString("F2", CultureInfo.InvariantCulture), + single.AverageAltStackDepth.ToString("F2", CultureInfo.InvariantCulture), + saturated.AverageAltStackDepth.ToString("F2", CultureInfo.InvariantCulture), + altStackDepthScale.ToString("F4", CultureInfo.InvariantCulture), + baseline.GasPerIteration.ToString("F6", CultureInfo.InvariantCulture), + single.GasPerIteration.ToString("F6", CultureInfo.InvariantCulture), + saturated.GasPerIteration.ToString("F6", CultureInfo.InvariantCulture), + gasIterationScale.ToString("F4", CultureInfo.InvariantCulture), + baseline.TotalDataBytes.ToString(CultureInfo.InvariantCulture), + single.TotalDataBytes.ToString(CultureInfo.InvariantCulture), + saturated.TotalDataBytes.ToString(CultureInfo.InvariantCulture), + baseline.TotalAllocatedBytes.ToString(CultureInfo.InvariantCulture), + single.TotalAllocatedBytes.ToString(CultureInfo.InvariantCulture), + saturated.TotalAllocatedBytes.ToString(CultureInfo.InvariantCulture), + baseline.TotalGasConsumed.ToString(CultureInfo.InvariantCulture), + single.TotalGasConsumed.ToString(CultureInfo.InvariantCulture), + saturated.TotalGasConsumed.ToString(CultureInfo.InvariantCulture))); + } + + return scalingPath; + } + + private sealed record ScalingMeasurement( + BenchmarkComponent Component, + string ScenarioId, + BenchmarkOperationKind OperationKind, + string OperationId, + ScenarioComplexity Complexity, + BenchmarkVariant Variant, + double NanosecondsPerIteration, + double NanosecondsPerByte, + double AllocatedBytesPerIteration, + double AllocatedBytesPerByte, + long TotalDataBytes, + long TotalAllocatedBytes, + double AverageStackDepth, + int PeakStackDepth, + double AverageAltStackDepth, + int PeakAltStackDepth, + long TotalGasConsumed, + double GasPerIteration) + { + public static bool TryParse(string line, out ScalingMeasurement measurement) + { + var parts = line.Split(',', StringSplitOptions.None); + if (parts.Length < 28) + { + measurement = default!; + return false; + } + + try + { + var component = Enum.Parse(parts[0], ignoreCase: false); + var scenarioId = parts[1]; + var operationKind = Enum.Parse(parts[2], ignoreCase: false); + var operationId = parts[3]; + var complexity = Enum.Parse(parts[4], ignoreCase: false); + var variant = Enum.Parse(parts[5], ignoreCase: false); + var totalDataBytes = long.Parse(parts[10], CultureInfo.InvariantCulture); + var nanosecondsPerIteration = double.Parse(parts[15], CultureInfo.InvariantCulture); + var nanosecondsPerByte = double.Parse(parts[16], CultureInfo.InvariantCulture); + var totalAllocatedBytes = long.Parse(parts[18], CultureInfo.InvariantCulture); + var allocatedPerIteration = double.Parse(parts[19], CultureInfo.InvariantCulture); + var allocatedPerByte = double.Parse(parts[20], CultureInfo.InvariantCulture); + var averageStack = double.Parse(parts[22], CultureInfo.InvariantCulture); + var peakStack = int.Parse(parts[23], CultureInfo.InvariantCulture); + var averageAltStack = double.Parse(parts[24], CultureInfo.InvariantCulture); + var peakAltStack = int.Parse(parts[25], CultureInfo.InvariantCulture); + var totalGas = long.Parse(parts[26], CultureInfo.InvariantCulture); + var gasPerIteration = double.Parse(parts[27], CultureInfo.InvariantCulture); + + measurement = new ScalingMeasurement(component, scenarioId, operationKind, operationId, complexity, variant, nanosecondsPerIteration, nanosecondsPerByte, allocatedPerIteration, allocatedPerByte, totalDataBytes, totalAllocatedBytes, averageStack, peakStack, averageAltStack, peakAltStack, totalGas, gasPerIteration); + return true; + } + catch + { + measurement = default!; + return false; + } + } + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkProtocolSettings.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkProtocolSettings.cs new file mode 100644 index 0000000000..c112c289f3 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkProtocolSettings.cs @@ -0,0 +1,92 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// BenchmarkProtocolSettings.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo; +using Neo.Cryptography.ECC; +using Neo.SmartContract; +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.Linq; + +namespace Neo.VM.Benchmark.Infrastructure +{ + internal static class BenchmarkProtocolSettings + { + private static readonly IReadOnlyList s_fallbackValidators = CreateFallbackValidators(); + private static readonly IReadOnlyList s_fallbackCommittee = s_fallbackValidators; + + public static IReadOnlyList StandbyValidators => ProtocolSettings.Default.StandbyValidators.Count > 0 + ? ProtocolSettings.Default.StandbyValidators + : s_fallbackValidators; + + public static IReadOnlyList StandbyCommittee => ProtocolSettings.Default.StandbyCommittee.Count > 0 + ? ProtocolSettings.Default.StandbyCommittee + : s_fallbackCommittee; + + public static int ValidatorsCount => ProtocolSettings.Default.ValidatorsCount > 0 + ? ProtocolSettings.Default.ValidatorsCount + : StandbyValidators.Count; + + public static UInt160 GetCommitteeAddress() + { + return Contract.GetBFTAddress(StandbyValidators); + } + + public static ProtocolSettings ResolveSettings(ProtocolSettings? candidate = null) + { + var baseSettings = candidate ?? ProtocolSettings.Default; + if (baseSettings.StandbyCommittee.Count > 0 && baseSettings.ValidatorsCount > 0) + return baseSettings; + + var committee = s_fallbackCommittee.Select(static p => (ECPoint)p).ToArray(); + var validatorsCount = committee.Length; + + return new ProtocolSettings + { + Network = baseSettings.Network, + AddressVersion = baseSettings.AddressVersion, + StandbyCommittee = committee, + ValidatorsCount = validatorsCount, + SeedList = baseSettings.SeedList.ToArray(), + MillisecondsPerBlock = baseSettings.MillisecondsPerBlock, + MaxValidUntilBlockIncrement = baseSettings.MaxValidUntilBlockIncrement, + MaxTransactionsPerBlock = baseSettings.MaxTransactionsPerBlock, + MemoryPoolMaxTransactions = baseSettings.MemoryPoolMaxTransactions, + MaxTraceableBlocks = baseSettings.MaxTraceableBlocks, + InitialGasDistribution = baseSettings.InitialGasDistribution, + Hardforks = baseSettings.Hardforks + }; + } + + private static IReadOnlyList CreateFallbackValidators() + { + var curve = ECCurve.Secp256r1; + return new[] + { + Multiply(curve.G, 1u), + Multiply(curve.G, 2u), + Multiply(curve.G, 3u), + Multiply(curve.G, 5u), + Multiply(curve.G, 7u), + Multiply(curve.G, 11u), + Multiply(curve.G, 13u) + }; + } + + private static ECPoint Multiply(ECPoint point, uint scalar) + { + var buffer = new byte[32]; + BinaryPrimitives.WriteUInt32BigEndian(buffer.AsSpan(28), scalar); + return point * buffer; + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkResultRecorder.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkResultRecorder.cs new file mode 100644 index 0000000000..be1f925d74 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/BenchmarkResultRecorder.cs @@ -0,0 +1,191 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// BenchmarkResultRecorder.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using System; +using System.Collections.Concurrent; +using System.Linq; + +namespace Neo.VM.Benchmark.Infrastructure +{ + /// + /// Captures benchmark metrics for VM scenarios with awareness of component, operation kind and workload profile. + /// + public sealed class BenchmarkResultRecorder + { + private readonly ConcurrentDictionary _measurements = new(); + + public void RecordInstruction(VM.OpCode opcode, TimeSpan elapsed, long allocatedBytes, int stackDepth, int altStackDepth, long gasConsumed) + { + Record(BenchmarkOperationKind.Instruction, opcode.ToString(), elapsed, allocatedBytes, stackDepth, altStackDepth, gasConsumed); + } + + public void RecordSyscall(string name, TimeSpan elapsed, long allocatedBytes, int stackDepth, int altStackDepth, long gasConsumed) + { + Record(BenchmarkOperationKind.Syscall, name, elapsed, allocatedBytes, stackDepth, altStackDepth, gasConsumed); + } + + public void RecordNativeMethod(string name, TimeSpan elapsed, long allocatedBytes, int stackDepth, int altStackDepth, long gasConsumed) + { + Record(BenchmarkOperationKind.NativeMethod, name, elapsed, allocatedBytes, stackDepth, altStackDepth, gasConsumed); + } + + public void Reset() + { + _measurements.Clear(); + } + + public IReadOnlyCollection Snapshot() + { + return _measurements.Select(static kvp => kvp.Key.ToMeasurement(kvp.Value)).ToArray(); + } + + private void Record(BenchmarkOperationKind kind, string operationId, TimeSpan elapsed, long allocatedBytes, int stackDepth, int altStackDepth, long gasConsumed) + { + var caseInfo = BenchmarkExecutionContext.CurrentCase; + var variant = BenchmarkExecutionContext.CurrentVariant; + if (caseInfo is null || variant is null) + return; + + var profile = BenchmarkExecutionContext.CurrentProfile ?? caseInfo.Profile; + + var key = new BenchmarkMeasurementKey( + caseInfo.Component, + caseInfo.Id, + operationId, + kind, + caseInfo.Complexity, + variant.Value, + profile); + + _measurements.AddOrUpdate( + key, + static (_, arg) => MeasurementAccumulator.Create(arg.Elapsed, arg.AllocatedBytes, arg.StackDepth, arg.AltStackDepth, arg.GasConsumed), + static (_, accumulator, arg) => accumulator.Add(arg.Elapsed, arg.AllocatedBytes, arg.StackDepth, arg.AltStackDepth, arg.GasConsumed), + (Elapsed: elapsed, AllocatedBytes: allocatedBytes, StackDepth: stackDepth, AltStackDepth: altStackDepth, GasConsumed: gasConsumed)); + } + + private readonly record struct MeasurementAccumulator(long Count, TimeSpan Total, long AllocatedBytes, long StackDepthTotal, int MaxStackDepth, long AltStackDepthTotal, int MaxAltStackDepth, long GasConsumedTotal) + { + public static MeasurementAccumulator Create(TimeSpan elapsed, long allocatedBytes, int stackDepth, int altStackDepth, long gasConsumed) + { + return new MeasurementAccumulator(1, elapsed, Math.Max(allocatedBytes, 0), stackDepth, stackDepth, altStackDepth, altStackDepth, Math.Max(gasConsumed, 0)); + } + + public MeasurementAccumulator Add(TimeSpan elapsed, long allocatedBytes, int stackDepth, int altStackDepth, long gasConsumed) + { + return new MeasurementAccumulator( + Count + 1, + Total + elapsed, + AllocatedBytes + Math.Max(allocatedBytes, 0), + StackDepthTotal + Math.Max(stackDepth, 0), + Math.Max(MaxStackDepth, stackDepth), + AltStackDepthTotal + Math.Max(altStackDepth, 0), + Math.Max(MaxAltStackDepth, altStackDepth), + GasConsumedTotal + Math.Max(gasConsumed, 0)); + } + } + + private readonly record struct BenchmarkMeasurementKey( + BenchmarkComponent Component, + string ScenarioId, + string OperationId, + BenchmarkOperationKind OperationKind, + ScenarioComplexity Complexity, + BenchmarkVariant Variant, + ScenarioProfile Profile) + { + public BenchmarkMeasurement ToMeasurement(MeasurementAccumulator accumulator) + { + return new BenchmarkMeasurement( + Component, + ScenarioId, + OperationId, + OperationKind, + Complexity, + Variant, + Profile, + accumulator.Count, + accumulator.Total, + accumulator.AllocatedBytes, + accumulator.StackDepthTotal, + accumulator.MaxStackDepth, + accumulator.AltStackDepthTotal, + accumulator.MaxAltStackDepth, + accumulator.GasConsumedTotal); + } + } + + public readonly record struct BenchmarkMeasurement( + BenchmarkComponent Component, + string ScenarioId, + string OperationId, + BenchmarkOperationKind OperationKind, + ScenarioComplexity Complexity, + BenchmarkVariant Variant, + ScenarioProfile Profile, + long Count, + TimeSpan Total, + long AllocatedBytes, + long StackDepthTotal, + int MaxStackDepth, + long AltStackDepthTotal, + int MaxAltStackDepth, + long GasConsumedTotal) + { + private double TotalNanoseconds => Total.TotalMilliseconds * 1_000_000.0; + public double TotalMicroseconds => TotalNanoseconds / 1000.0; + public double AverageNanoseconds => Count > 0 ? TotalNanoseconds / Count : 0; + + public double NanosecondsPerIteration + { + get + { + var iterations = TotalIterations; + return TotalNanoseconds / iterations; + } + } + + public double NanosecondsPerByte + { + get + { + var totalBytes = TotalDataBytes; + return totalBytes > 0 ? TotalNanoseconds / totalBytes : 0; + } + } + + public double NanosecondsPerElement + { + get + { + var totalElements = TotalElements; + return totalElements > 0 ? TotalNanoseconds / totalElements : 0; + } + } + + public long TotalIterations => Math.Max(1, Profile.Iterations); + public long TotalDataBytes => (long)Math.Max(0, Profile.DataLength) * TotalIterations; + public long TotalElements => (long)Math.Max(0, Profile.CollectionLength) * TotalIterations; + public long TotalAllocatedBytes => AllocatedBytes; + + public double AllocatedBytesPerIteration => TotalIterations > 0 ? (double)TotalAllocatedBytes / TotalIterations : 0; + public double AllocatedBytesPerByte => TotalDataBytes > 0 ? (double)TotalAllocatedBytes / TotalDataBytes : 0; + public double AllocatedBytesPerElement => TotalElements > 0 ? (double)TotalAllocatedBytes / TotalElements : 0; + + public double AverageStackDepth => Count > 0 ? (double)StackDepthTotal / Count : 0; + public int PeakStackDepth => MaxStackDepth; + public double AverageAltStackDepth => Count > 0 ? (double)AltStackDepthTotal / Count : 0; + public int PeakAltStackDepth => MaxAltStackDepth; + public long TotalGasConsumed => GasConsumedTotal; + public double GasPerIteration => TotalIterations > 0 ? (double)TotalGasConsumed / TotalIterations : 0; + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/DelegatingScenario.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/DelegatingScenario.cs new file mode 100644 index 0000000000..88c28a1e42 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/DelegatingScenario.cs @@ -0,0 +1,43 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// DelegatingScenario.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +namespace Neo.VM.Benchmark.Infrastructure +{ + /// + /// Convenience scenario that delegates execution to pre-defined callbacks per variant. + /// + public sealed class DelegatingScenario : IVmScenario + { + private readonly IReadOnlyDictionary _actions; + + public DelegatingScenario(IReadOnlyDictionary actions) + { + _actions = actions; + } + + public void Execute(BenchmarkVariant variant) + { + if (_actions.TryGetValue(variant, out var action)) + { + action(); + return; + } + + if (_actions.TryGetValue(BenchmarkVariant.Single, out var fallback)) + { + fallback(); + return; + } + + throw new NotSupportedException($"Scenario variant '{variant}' is not supported."); + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/IVmScenario.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/IVmScenario.cs new file mode 100644 index 0000000000..f5d5dc6955 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/IVmScenario.cs @@ -0,0 +1,21 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// IVmScenario.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +namespace Neo.VM.Benchmark.Infrastructure +{ + /// + /// Abstraction for an executable benchmark scenario. + /// + public interface IVmScenario + { + void Execute(BenchmarkVariant variant); + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/InteropCoverageReport.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/InteropCoverageReport.cs new file mode 100644 index 0000000000..5d68c1787d --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/InteropCoverageReport.cs @@ -0,0 +1,54 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// InteropCoverageReport.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo.SmartContract; +using Neo.SmartContract.Native; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Neo.VM.Benchmark.Infrastructure +{ + internal static class InteropCoverageReport + { + public static IReadOnlyCollection MissingSyscalls(IReadOnlyCollection covered) + { + var all = ApplicationEngine.Services.Values + .Where(d => d.Hardfork is null) + .Select(d => d.Name) + .ToHashSet(StringComparer.Ordinal); + all.ExceptWith(covered); + return all.OrderBy(n => n, StringComparer.Ordinal).ToArray(); + } + + public static IReadOnlyCollection MissingNativeMethods(IReadOnlyCollection covered) + { + var all = NativeContract.Contracts + .SelectMany(contract => contract.GetContractState(ProtocolSettings.Default, 0).Manifest.Abi.Methods + .Select(m => $"{contract.Name}:{m.Name}")) + .ToHashSet(StringComparer.Ordinal); + all.ExceptWith(covered); + return all.OrderBy(n => n, StringComparer.Ordinal).ToArray(); + } + + public static void WriteReport(string path, IReadOnlyCollection missingSyscalls, IReadOnlyCollection missingNatives) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + using var writer = new StreamWriter(path, append: false); + writer.WriteLine("Category,Identifier"); + foreach (var syscall in missingSyscalls) + writer.WriteLine($"syscall,{syscall}"); + foreach (var native in missingNatives) + writer.WriteLine($"native,{native}"); + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/LoopScriptFactory.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/LoopScriptFactory.cs new file mode 100644 index 0000000000..9e68ace444 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/LoopScriptFactory.cs @@ -0,0 +1,67 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// LoopScriptFactory.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +namespace Neo.VM.Benchmark.Infrastructure +{ + /// + /// Utility to compose loop-based scripts for opcode scenarios. + /// + internal static class LoopScriptFactory + { + public static byte[] BuildCountingLoop(ScenarioProfile profile, Action iteration, byte localCount = 1, byte argumentCount = 0) => + BuildCountingLoop(profile, null, iteration, null, localCount, argumentCount); + + public static byte[] BuildCountingLoop( + ScenarioProfile profile, + Action? prolog, + Action iteration, + Action? epilog = null, + byte localCount = 1, + byte argumentCount = 0) + { + var builder = new InstructionBuilder(); + builder.AddInstruction(new Instruction { _opCode = VM.OpCode.INITSLOT, _operand = new[] { localCount, argumentCount } }); + builder.Push(profile.Iterations); + builder.AddInstruction(VM.OpCode.STLOC0); + + prolog?.Invoke(builder); + + var loopStart = new JumpTarget { _instruction = builder.AddInstruction(VM.OpCode.NOP) }; + iteration(builder); + builder.AddInstruction(VM.OpCode.LDLOC0); + builder.AddInstruction(VM.OpCode.DEC); + builder.AddInstruction(VM.OpCode.DUP); + builder.AddInstruction(VM.OpCode.STLOC0); + builder.Jump(VM.OpCode.JMPIF_L, loopStart); + epilog?.Invoke(builder); + builder.AddInstruction(VM.OpCode.RET); + return builder.ToArray(); + } + + public static byte[] BuildInfiniteLoop(Action iteration) => + BuildInfiniteLoop(null, iteration, null); + + public static byte[] BuildInfiniteLoop( + Action? prolog, + Action iteration, + Action? epilog = null) + { + var builder = new InstructionBuilder(); + prolog?.Invoke(builder); + var loopStart = new JumpTarget { _instruction = builder.AddInstruction(VM.OpCode.NOP) }; + iteration(builder); + epilog?.Invoke(builder); + builder.Jump(VM.OpCode.JMP_L, loopStart); + builder.AddInstruction(VM.OpCode.RET); + return builder.ToArray(); + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/ManualSuiteRunner.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/ManualSuiteRunner.cs new file mode 100644 index 0000000000..233d0187ad --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/ManualSuiteRunner.cs @@ -0,0 +1,58 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// ManualSuiteRunner.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using BenchmarkDotNet.Attributes; +using System; +using System.Linq; +using System.Reflection; + +namespace Neo.VM.Benchmark.Infrastructure +{ + internal static class ManualSuiteRunner + { + public static void RunAll() + { + RunSuite(new Syscalls.SyscallSuite()); + RunSuite(new Native.NativeSuite()); + RunSuite(new OpCode.OpcodeSuite()); + } + + private static void RunSuite(TSuite suite) where TSuite : VmBenchmarkSuite + { + var type = suite.GetType(); + InvokeLifecycle(type, suite, typeof(GlobalSetupAttribute)); + + foreach (var vmCase in suite.Cases()) + { + suite.Case = vmCase; + InvokeLifecycle(type, suite, typeof(IterationSetupAttribute)); + suite.Baseline(); + InvokeLifecycle(type, suite, typeof(IterationSetupAttribute)); + suite.Single(); + InvokeLifecycle(type, suite, typeof(IterationSetupAttribute)); + suite.Saturated(); + } + + InvokeLifecycle(type, suite, typeof(GlobalCleanupAttribute)); + } + + private static void InvokeLifecycle(Type suiteType, object instance, Type attributeType) + { + var methods = suiteType + .GetMethods(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .Where(m => m.GetCustomAttributes(attributeType, inherit: false).Any()); + foreach (var method in methods) + { + method.Invoke(instance, null); + } + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/ManualWitness.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/ManualWitness.cs new file mode 100644 index 0000000000..dc7a634e58 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/ManualWitness.cs @@ -0,0 +1,55 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// ManualWitness.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo.IO; +using Neo.Network.P2P.Payloads; +using Neo.Persistence; +using Neo.SmartContract; +using System; +using System.IO; + +namespace Neo.VM.Benchmark.Infrastructure +{ + /// + /// Lightweight implementation that advertises pre-defined witnesses. + /// + internal sealed class ManualWitness : IVerifiable + { + private readonly UInt160[] _hashes; + + public ManualWitness(params UInt160[] hashes) + { + _hashes = hashes ?? Array.Empty(); + } + + public int Size => 0; + + public Witness[] Witnesses { get; set; } = Array.Empty(); + + public void Deserialize(ref MemoryReader reader) + { + } + + public void DeserializeUnsigned(ref MemoryReader reader) + { + } + + public UInt160[] GetScriptHashesForVerifying(DataCache snapshot) => _hashes; + + public void Serialize(BinaryWriter writer) + { + } + + public void SerializeUnsigned(BinaryWriter writer) + { + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/NativeBenchmarkStateFactory.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/NativeBenchmarkStateFactory.cs new file mode 100644 index 0000000000..9ac0835195 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/NativeBenchmarkStateFactory.cs @@ -0,0 +1,54 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeBenchmarkStateFactory.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo; +using Neo.Network.P2P.Payloads; +using Neo.Persistence; +using Neo.Persistence.Providers; +using Neo.SmartContract; +using System; + +namespace Neo.VM.Benchmark.Infrastructure +{ + /// + /// Provides shared blockchain state for native contract benchmarks. + /// + internal static class NativeBenchmarkStateFactory + { + private sealed class BenchmarkStoreProvider : IStoreProvider + { + public readonly MemoryStore Store = new(); + + public string Name => "BenchmarkMemoryStore"; + + public IStore GetStore(string path) => Store; + } + + private static readonly Lazy<(NeoSystem System, BenchmarkStoreProvider Provider)> s_system = new(() => + { + var provider = new BenchmarkStoreProvider(); + var settings = BenchmarkProtocolSettings.ResolveSettings(); + var system = new NeoSystem(settings, provider); + system.SuspendNodeStartup(); + return (system, provider); + }); + + public static StoreCache CreateSnapshot() + { + return s_system.Value.System.GetSnapshotCache(); + } + + public static BenchmarkApplicationEngine CreateEngine(IVerifiable? container = null) + { + return BenchmarkApplicationEngine.Create(snapshot: CreateSnapshot(), container: container); + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/OpcodeVmScenario.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/OpcodeVmScenario.cs new file mode 100644 index 0000000000..992cfe588c --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/OpcodeVmScenario.cs @@ -0,0 +1,72 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// OpcodeVmScenario.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo.VM.Benchmark.OpCode; +using System; + +namespace Neo.VM.Benchmark.Infrastructure +{ + /// + /// Scenario wrapper executing pre-generated opcode scripts. + /// + public sealed class OpcodeVmScenario : IVmScenario + { + private readonly byte[] _baselineScript; + private readonly byte[] _singleScript; + private readonly byte[] _saturatedScript; + private readonly long _saturatedGasBudget; + private readonly Action? _before; + private readonly Action? _after; + + public OpcodeVmScenario( + byte[] baselineScript, + byte[] singleScript, + byte[] saturatedScript, + long saturatedGasBudget, + Action? before = null, + Action? after = null) + { + _baselineScript = baselineScript; + _singleScript = singleScript; + _saturatedScript = saturatedScript; + _saturatedGasBudget = saturatedGasBudget; + _before = before; + _after = after; + } + + public void Execute(BenchmarkVariant variant) + { + switch (variant) + { + case BenchmarkVariant.Baseline: + ExecuteScript(_baselineScript, static (engine, _) => engine.ExecuteBenchmark()); + break; + case BenchmarkVariant.Single: + ExecuteScript(_singleScript, static (engine, _) => engine.ExecuteBenchmark()); + break; + case BenchmarkVariant.Saturated: + ExecuteScript(_saturatedScript, (engine, _) => engine.ExecuteUntilGas(_saturatedGasBudget)); + break; + default: + throw new ArgumentOutOfRangeException(nameof(variant), variant, null); + } + } + + private void ExecuteScript(byte[] script, Action runner) + { + using var engine = Benchmark_Opcode.LoadScript(script); + engine.BeforeInstruction = _before; + engine.AfterInstruction = _after; + engine.Recorder = BenchmarkExecutionContext.CurrentRecorder; + runner(engine, script); + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/ScenarioProfiles.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/ScenarioProfiles.cs new file mode 100644 index 0000000000..e903957d68 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/ScenarioProfiles.cs @@ -0,0 +1,71 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// ScenarioProfiles.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using System; + +namespace Neo.VM.Benchmark.Infrastructure +{ + /// + /// Describes the set of standard workload tiers used by benchmarks. + /// + public enum ScenarioComplexity + { + Micro, + Standard, + Stress + } + + /// + /// Structural data for a workload profile. + /// + public readonly record struct ScenarioProfile(int Iterations, int DataLength, int CollectionLength) + { + public static ScenarioProfile For(ScenarioComplexity complexity) => complexity switch + { + ScenarioComplexity.Micro => new ScenarioProfile(Iterations: 8, DataLength: 32, CollectionLength: 8), + ScenarioComplexity.Standard => new ScenarioProfile(Iterations: 64, DataLength: 512, CollectionLength: 64), + ScenarioComplexity.Stress => new ScenarioProfile(Iterations: 512, DataLength: 16 * 1024, CollectionLength: 256), + _ => throw new ArgumentOutOfRangeException(nameof(complexity), complexity, null) + }; + + public bool IsEmpty => Iterations == 0 && DataLength == 0 && CollectionLength == 0; + + public ScenarioProfile With(int? iterations = null, int? dataLength = null, int? collectionLength = null) + { + return new ScenarioProfile( + iterations ?? Iterations, + dataLength ?? DataLength, + collectionLength ?? CollectionLength); + } + + public ScenarioProfile Scale(double iterationFactor = 1d, double dataFactor = 1d, double collectionFactor = 1d) + { + static int ScaleValue(int value, double factor) => factor switch + { + 1d => value, + _ => (int)Math.Max(0, Math.Round(value * factor, MidpointRounding.AwayFromZero)) + }; + + return new ScenarioProfile( + ScaleValue(Iterations, iterationFactor), + ScaleValue(DataLength, dataFactor), + ScaleValue(CollectionLength, collectionFactor)); + } + + public ScenarioProfile EnsureMinimum(int minIterations = 0, int minDataLength = 0, int minCollectionLength = 0) + { + return new ScenarioProfile( + Math.Max(Iterations, minIterations), + Math.Max(DataLength, minDataLength), + Math.Max(CollectionLength, minCollectionLength)); + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/VmBenchmarkCase.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/VmBenchmarkCase.cs new file mode 100644 index 0000000000..0f7e51ec91 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/VmBenchmarkCase.cs @@ -0,0 +1,39 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// VmBenchmarkCase.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +namespace Neo.VM.Benchmark.Infrastructure +{ + /// + /// Aggregates the metadata and scenario used by BenchmarkDotNet parameter sources. + /// + public sealed class VmBenchmarkCase + { + public string Id { get; } + public BenchmarkComponent Component { get; } + public ScenarioComplexity Complexity { get; } + public ScenarioProfile Profile { get; } + public IVmScenario Scenario { get; } + + public VmBenchmarkCase(string id, BenchmarkComponent component, ScenarioComplexity complexity, IVmScenario scenario, ScenarioProfile? profileOverride = null) + { + Id = id; + Component = component; + Complexity = complexity; + Profile = profileOverride ?? ScenarioProfile.For(complexity); + Scenario = scenario; + } + + public override string ToString() + { + return $"{Component}:{Id} ({Complexity})"; + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/infrastructure/VmBenchmarkSuite.cs b/benchmarks/Neo.VM.Benchmarks/infrastructure/VmBenchmarkSuite.cs new file mode 100644 index 0000000000..d9d3cbbed8 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/infrastructure/VmBenchmarkSuite.cs @@ -0,0 +1,55 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// VmBenchmarkSuite.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using BenchmarkDotNet.Attributes; +using System; + +namespace Neo.VM.Benchmark.Infrastructure +{ + /// + /// Base class for BenchmarkDotNet suites that execute VM scenarios. + /// + public abstract class VmBenchmarkSuite + { + [ParamsSource(nameof(Cases))] + public VmBenchmarkCase Case { get; set; } = default!; + + protected abstract IEnumerable GetCases(); + + public IEnumerable Cases() => GetCases(); + + [Benchmark(Baseline = true)] + public void Baseline() => RunVariant(BenchmarkVariant.Baseline); + + [Benchmark] + public void Single() => RunVariant(BenchmarkVariant.Single); + + [Benchmark] + public void Saturated() => RunVariant(BenchmarkVariant.Saturated); + + private void RunVariant(BenchmarkVariant variant) + { + if (Case is null) + throw new InvalidOperationException("Benchmark case not initialized."); + + BenchmarkExecutionContext.CurrentCase = Case; + BenchmarkExecutionContext.CurrentVariant = variant; + try + { + Case.Scenario.Execute(variant); + } + finally + { + BenchmarkExecutionContext.ClearVariant(); + } + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/native/NativeCoverageReport.cs b/benchmarks/Neo.VM.Benchmarks/native/NativeCoverageReport.cs new file mode 100644 index 0000000000..6a0ebb60bf --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/native/NativeCoverageReport.cs @@ -0,0 +1,27 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeCoverageReport.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo.SmartContract.Native; +using Neo.VM.Benchmark.Infrastructure; +using System.Collections.Generic; +using System.Linq; + +namespace Neo.VM.Benchmark.Native +{ + internal static class NativeCoverageReport + { + public static IReadOnlyCollection GetMissing() + { + var covered = NativeCoverageTracker.GetCovered(); + return InteropCoverageReport.MissingNativeMethods(covered); + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/native/NativeCoverageTracker.cs b/benchmarks/Neo.VM.Benchmarks/native/NativeCoverageTracker.cs new file mode 100644 index 0000000000..7b982ba708 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/native/NativeCoverageTracker.cs @@ -0,0 +1,37 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeCoverageTracker.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using System.Collections.Generic; +using System.Linq; + +namespace Neo.VM.Benchmark.Native +{ + internal static class NativeCoverageTracker + { + private static readonly HashSet s_covered = new(); + + public static void Register(string id) + { + lock (s_covered) + { + s_covered.Add(id); + } + } + + public static IReadOnlyCollection GetCovered() + { + lock (s_covered) + { + return s_covered.ToArray(); + } + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/native/NativeScenarioFactory.cs b/benchmarks/Neo.VM.Benchmarks/native/NativeScenarioFactory.cs new file mode 100644 index 0000000000..bcb3a6c172 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/native/NativeScenarioFactory.cs @@ -0,0 +1,1854 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeScenarioFactory.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo; +using Neo.Cryptography; +using Neo.Cryptography.ECC; +using Neo.Extensions; +using Neo.Json; +using Neo.Network.P2P.Payloads; +using Neo.Persistence; +using Neo.SmartContract; +using Neo.SmartContract.Manifest; +using Neo.SmartContract.Native; +using Neo.VM; +using Neo.VM.Benchmark.Infrastructure; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Numerics; +using System.Reflection; +using System.Text; +using VMArray = Neo.VM.Types.Array; +using VMStackItem = Neo.VM.Types.StackItem; + +namespace Neo.VM.Benchmark.Native +{ + /// + /// Builds simple read-only scenarios for native contract benchmarking. + /// + internal static class NativeScenarioFactory + { + private sealed record NativeMethod( + string Id, + UInt160 ContractHash, + string MethodName, + CallFlags Flags, + Action? EmitArguments = null, + bool DropResult = true, + Func? EngineFactory = null, + Action? ConfigureEngine = null, + Func? ProfileFactory = null, + Func? ScriptFactory = null); + + private static readonly IReadOnlyList s_standbyValidators = BenchmarkProtocolSettings.StandbyValidators; + private static readonly IReadOnlyList s_standbyCommittee = BenchmarkProtocolSettings.StandbyCommittee; + + private static readonly UInt160 CommitteeAddress = Contract.GetBFTAddress(s_standbyValidators); + + private static readonly byte[] s_blsG1 = Convert.FromHexString("97F1D3A73197D7942695638C4FA9AC0FC3688C4F9774B905A14E3A3F171BAC586C55E83FF97A1AEFFB3AF00ADB22C6BB"); + private static readonly byte[] s_blsG2 = Convert.FromHexString("93E02B6052719F607DACD3A088274F65596BD0D09920B61AB5DA61BBDC7F5049334CF11213945D57E5AC7D055D042B7E024AA2B2F08F0A91260805272DC51051C6E47AD4FA403B02B4510B647AE3D1770BAC0326A805BBEFD48056C8C121BDB8"); + private static readonly byte[] s_blsGt = Convert.FromHexString("0F41E58663BF08CF068672CBD01A7EC73BACA4D72CA93544DEFF686BFD6DF543D48EAA24AFE47E1EFDE449383B67663104C581234D086A9902249B64728FFD21A189E87935A954051C7CDBA7B3872629A4FAFC05066245CB9108F0242D0FE3EF03350F55A7AEFCD3C31B4FCB6CE5771CC6A0E9786AB5973320C806AD360829107BA810C5A09FFDD9BE2291A0C25A99A211B8B424CD48BF38FCEF68083B0B0EC5C81A93B330EE1A677D0D15FF7B984E8978EF48881E32FAC91B93B47333E2BA5706FBA23EB7C5AF0D9F80940CA771B6FFD5857BAAF222EB95A7D2809D61BFE02E1BFD1B68FF02F0B8102AE1C2D5D5AB1A19F26337D205FB469CD6BD15C3D5A04DC88784FBB3D0B2DBDEA54D43B2B73F2CBB12D58386A8703E0F948226E47EE89D018107154F25A764BD3C79937A45B84546DA634B8F6BE14A8061E55CCEBA478B23F7DACAA35C8CA78BEAE9624045B4B601B2F522473D171391125BA84DC4007CFBF2F8DA752F7C74185203FCCA589AC719C34DFFBBAAD8431DAD1C1FB597AAA5193502B86EDB8857C273FA075A50512937E0794E1E65A7617C90D8BD66065B1FFFE51D7A579973B1315021EC3C19934F1368BB445C7C2D209703F239689CE34C0378A68E72A6B3B216DA0E22A5031B54DDFF57309396B38C881C4C849EC23E87089A1C5B46E5110B86750EC6A532348868A84045483C92B7AF5AF689452EAFABF1A8943E50439F1D59882A98EAA0170F1250EBD871FC0A92A7B2D83168D0D727272D441BEFA15C503DD8E90CE98DB3E7B6D194F60839C508A84305AACA1789B6"); + private static readonly byte[] s_blsScalar = CreateBlsScalar(); + private static readonly byte[] s_recoverMessageHash = Convert.FromHexString("5AE8317D34D1E595E3FA7247DB80C0AF4320CCE1116DE187F8F7E2E099C0D8D0"); + private static readonly byte[] s_recoverSignature = Convert.FromHexString("45C0B7F8C09A9E1F1CEA0C25785594427B6BF8F9F878A8AF0B1ABBB48E16D0920D8BECD0C220F67C51217EECFD7184EF0732481C843857E6BC7FC095C4F6B78801"); + private static readonly byte[] s_verifyMessage = Encoding.UTF8.GetBytes("中文"); + private static readonly byte[] s_verifySignature = Convert.FromHexString("B8CBA1FF42304D74D083E87706058F59CDD4F755B995926D2CD80A734C5A3C37E4583BFD4339AC762C1C91EEE3782660A6BAF62CD29E407ECCD3DA3E9DE55A02"); + private static readonly byte[] s_verifyPubKey = Convert.FromHexString("03661B86D54EB3A8E7EA2399E0DB36AB65753F95FFF661DA53AE0121278B881AD0"); + private static readonly byte[] s_ed25519Signature = Convert.FromHexString("E5564300C360AC729086E2CC806E828A84877F1EB8E5D974D873E065224901555FB8821590A33BACC61E39701CF9B46BD25BF5F0595BBE24655141438E7A100B"); + private static readonly byte[] s_ed25519PublicKey = Convert.FromHexString("D75A980182B10AB7D54BFED3C964073A0EE172F3DAA62325AF021A68F707511A"); + private static readonly ECPoint s_committeePublicKey = GetCommitteePublicKey(); + private const byte TokenAccountPrefix = 0x14; + private const byte PolicyBlockedAccountPrefix = 15; + private const byte ContractManagementStoragePrefixContract = 8; + private const byte ContractManagementStoragePrefixContractHash = 12; + private const byte NeoAccountPrefix = 20; + private const byte NeoCandidatePrefix = 33; + private const byte NeoRegisterPricePrefix = 13; + private const byte NeoVotersCountPrefix = 1; + private const byte NeoGasPerBlockPrefix = 29; + private const byte NeoVoterRewardPrefix = 23; + private const byte NotaryDepositPrefix = 1; + private const byte NotaryMaxNotValidBeforeDeltaPrefix = 10; + private const int NotaryDefaultDepositDeltaTill = 5760; + private const int NotaryDefaultMaxNotValidBeforeDelta = 140; + private const int OracleMaxUserDataLength = 512; + private const byte OraclePricePrefix = 5; + private const byte OracleRequestIdPrefix = 9; + private const byte OracleRequestPrefix = 7; + private const byte OracleIdListPrefix = 6; + private static readonly byte[] s_committeePublicKeyCompressed = s_committeePublicKey.EncodePoint(true); + private static readonly int s_validatorsCount = BenchmarkProtocolSettings.ValidatorsCount; + private static readonly UInt160 s_gasTransferRecipient = UInt160.Parse("0x0102030405060708090a0b0c0d0e0f1011121314"); + private static readonly BigInteger s_gasTransferAmount = new BigInteger(1_0000_0000); + private static readonly byte[] s_zeroHashBytes = new byte[UInt256.Length]; + private static readonly byte[] s_blockIndexBytes = [0x00]; + private static readonly long s_policyFeePerByteValue = PolicyContract.DefaultFeePerByte; + private static readonly uint s_policyExecFeeFactorValue = PolicyContract.DefaultExecFeeFactor; + private static readonly uint s_policyStoragePriceValue = PolicyContract.DefaultStoragePrice; + private static readonly uint s_policyMillisecondsPerBlockValue = (uint)ProtocolSettings.Default.MillisecondsPerBlock; + private static readonly uint s_policyMaxValidUntilValue = Math.Max(1u, Math.Min(PolicyContract.MaxMaxValidUntilBlockIncrement, ProtocolSettings.Default.MaxValidUntilBlockIncrement)); + private static readonly uint s_policyMaxTraceableValue = Math.Min(PolicyContract.MaxMaxTraceableBlocks, Math.Max(s_policyMaxValidUntilValue + 1, ProtocolSettings.Default.MaxTraceableBlocks)); + private static readonly uint s_policyAttributeFeeValue = 1_0000u; + private static readonly byte s_policyAttributeType = (byte)TransactionAttributeType.HighPriority; + private static readonly BigInteger s_defaultRegisterPrice = 1000 * NativeContract.GAS.Factor; + + private sealed record ContractArtifact(ContractState State, byte[] NefBytes, byte[] ManifestBytes); + + private static readonly Action s_setCallFlagsAll = state => state.CallFlags = CallFlags.All; + private static readonly UInt160 s_contractDeployer = UInt160.Parse("0x0102030405060708090a0b0c0d0e0f1011121315"); + + private const string UpdateContractName = "BenchmarkContract.Update"; + + private static byte[] CreateBytePayload(ScenarioProfile profile, byte seed = 0x42, double scale = 1d, int minLength = 1) + { + var baseLength = profile.DataLength > 0 ? profile.DataLength : minLength; + var length = Math.Max(minLength, (int)Math.Round(baseLength * scale, MidpointRounding.AwayFromZero)); + return BenchmarkDataFactory.CreateByteArray(length, seed); + } + + private static string CreateStringPayload(ScenarioProfile profile, double scale = 1d, char seed = 'a', int minLength = 1) + { + var baseLength = profile.DataLength > 0 ? profile.DataLength : minLength; + var length = Math.Max(minLength, (int)Math.Round(baseLength * scale, MidpointRounding.AwayFromZero)); + return BenchmarkDataFactory.CreateString(length, seed); + } + + private static string CreateNumericPayload(ScenarioProfile profile, int minLength = 1) + { + var baseLength = profile.DataLength > 0 ? profile.DataLength : minLength; + return BenchmarkDataFactory.CreateNumericString(Math.Max(minLength, baseLength)); + } + + private static string CreateDelimitedString(ScenarioProfile profile, char delimiter = '-') + { + var segments = Math.Max(1, profile.CollectionLength); + var segmentLength = Math.Max(1, profile.DataLength / segments); + var builder = new StringBuilder(segments * (segmentLength + 1)); + for (int i = 0; i < segments; i++) + { + if (i > 0) + builder.Append(delimiter); + builder.Append(BenchmarkDataFactory.CreateString(segmentLength, (char)('a' + (i % 26)))); + } + if (builder.Length == 0) + return BenchmarkDataFactory.CreateString(Math.Max(1, profile.DataLength), 'a'); + return builder.ToString(); + } + + private static string CreateJsonPayload(ScenarioProfile profile) + { + var value = Math.Max(1, profile.DataLength); + return $"{{\"value\":{value}}}"; + } + + private static readonly (NefFile Nef, ContractManifest Manifest, byte[] NefBytes, byte[] ManifestBytes) s_updateTargetDocument = + CreateContractDocument(UpdateContractName, BuildSimpleScript(builder => builder.Push(2))); + private static readonly byte[] s_updateTargetNefBytes = s_updateTargetDocument.NefBytes; + private static readonly byte[] s_updateTargetManifestBytes = s_updateTargetDocument.ManifestBytes; + + private static readonly ContractArtifact s_deployContractArtifact = + CreateContractArtifact("BenchmarkContract.Deploy", BuildSimpleScript(builder => builder.Push(true)), 0x5000); + + private static readonly ContractArtifact s_updateContractArtifact = + CreateContractArtifact(UpdateContractName, + BuildContractManagementInvocationScript("update", CallFlags.States | CallFlags.AllowNotify, EmitContractUpdateArguments), + 0x5001); + + private static readonly ContractArtifact s_destroyContractArtifact = + CreateContractArtifact("BenchmarkContract.Destroy", + BuildContractManagementInvocationScript("destroy", CallFlags.States | CallFlags.AllowNotify, null), + 0x5002); + + private static readonly ContractArtifact s_oracleCallbackContract = + CreateContractArtifact("BenchmarkContract.OracleCallback", BuildSimpleScript(builder => { }), 0x5003); + + private static readonly UInt160 s_neoSenderAccount = UInt160.Parse("0xaabbccddeeff0011223344556677889900aabbcc"); + private static readonly UInt160 s_neoRecipientAccount = UInt160.Parse("0x11223344556677889900aabbccddeeff00112233"); + private static readonly ECPoint s_candidatePublicKey = s_standbyValidators.Count > 0 + ? s_standbyValidators[0] + : s_standbyCommittee.Count > 0 + ? s_standbyCommittee[0] + : ECCurve.Secp256r1.G; + private static readonly UInt160 s_candidateAccount = Contract.CreateSignatureRedeemScript(s_candidatePublicKey).ToScriptHash(); + private static readonly UInt160 s_notaryAccount = UInt160.Parse("0x44556677889900aabbccddeeff00112233445566"); + private static readonly UInt160 s_notaryRecipientAccount = UInt160.Parse("0x556677889900aabbccddeeff0011223344556677"); + private static readonly uint s_notaryInitialTill = 100u; + private static readonly uint s_notaryLockTargetTill = 200u; + private static readonly byte[] s_notaryDummySignature = new byte[64]; + private static readonly uint s_roleDesignationIndex = 1u; + private static readonly ECPoint[] s_roleDesignationNodes = s_standbyValidators.Take(1).ToArray(); + private const string OracleSampleUrl = "https://example.com/api"; + private const string OracleSampleFilter = "*"; + private const string OracleSampleCallback = "main"; + private static readonly byte[] s_oracleSampleResult = System.Array.Empty(); + private static readonly ulong s_oracleRequestId = 1uL; + + private static readonly PropertyInfo? s_nativeCallingScriptHashProperty = typeof(ExecutionContextState).GetProperty("NativeCallingScriptHash", BindingFlags.Instance | BindingFlags.NonPublic); + private static readonly Type? s_candidateStateType = typeof(NeoToken).GetNestedType("CandidateState", BindingFlags.NonPublic); + private static readonly FieldInfo? s_candidateRegisteredField = s_candidateStateType?.GetField("Registered", BindingFlags.Public | BindingFlags.Instance); + private static readonly FieldInfo? s_candidateVotesField = s_candidateStateType?.GetField("Votes", BindingFlags.Public | BindingFlags.Instance); + private static readonly MethodInfo? s_storageItemGetInteroperable = typeof(StorageItem).GetMethods(BindingFlags.Instance | BindingFlags.Public) + .FirstOrDefault(m => m.IsGenericMethodDefinition && m.Name == "GetInteroperable" && m.GetParameters().Length == 0); + private static readonly Type? s_roleNodeListType = typeof(RoleManagement).GetNestedType("NodeList", BindingFlags.NonPublic); + private static readonly MethodInfo? s_nodeListAddRangeMethod = s_roleNodeListType?.GetMethod("AddRange", BindingFlags.Public | BindingFlags.Instance); + private static readonly MethodInfo? s_nodeListSortMethod = s_roleNodeListType?.GetMethod("Sort", BindingFlags.Public | BindingFlags.Instance); + private static readonly Type? s_oracleIdListType = typeof(OracleContract).GetNestedType("IdList", BindingFlags.NonPublic); + private static readonly MethodInfo? s_idListAddRangeMethod = s_oracleIdListType?.GetMethod("AddRange", BindingFlags.Public | BindingFlags.Instance); + + private static NativeMethod CreateNativeMethod( + NativeContract contract, + string methodName, + CallFlags flags, + Action? emitArguments = null, + bool dropResult = true, + Func? engineFactory = null, + Action? configureEngine = null, + Func? profileFactory = null, + Func? scriptFactory = null) + { + return new NativeMethod($"{contract.Name}:{methodName}", contract.Hash, methodName, flags, emitArguments, dropResult, engineFactory, configureEngine, profileFactory, scriptFactory); + } + private static readonly IReadOnlyList Methods = new List + { + CreateNativeMethod(NativeContract.Policy, "getExecFeeFactor", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.Policy, "getStoragePrice", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.Policy, "getFeePerByte", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.Policy, "getMillisecondsPerBlock", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.Policy, "getMaxValidUntilBlockIncrement", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.Policy, "getMaxTraceableBlocks", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.Policy, "getAttributeFee", CallFlags.ReadStates, EmitPolicyAttributeFeeArguments), + CreateNativeMethod(NativeContract.Policy, "getBlockedAccounts", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.Policy, "isBlocked", CallFlags.ReadStates, EmitPolicyIsBlockedArguments), + CreateNativeMethod(NativeContract.Ledger, "currentIndex", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.Ledger, "currentHash", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.Ledger, "getBlock", CallFlags.ReadStates, EmitLedgerGetBlockArguments, dropResult: true), + CreateNativeMethod(NativeContract.Ledger, "getTransaction", CallFlags.ReadStates, EmitLedgerGetTransactionArguments, dropResult: true), + CreateNativeMethod(NativeContract.Ledger, "getTransactionFromBlock", CallFlags.ReadStates, EmitLedgerGetTransactionFromBlockArguments, dropResult: true), + CreateNativeMethod(NativeContract.Ledger, "getTransactionHeight", CallFlags.ReadStates, EmitLedgerGetTransactionHeightArguments, dropResult: true), + CreateNativeMethod(NativeContract.Ledger, "getTransactionSigners", CallFlags.ReadStates, EmitLedgerGetTransactionSignersArguments, dropResult: true), + CreateNativeMethod(NativeContract.Ledger, "getTransactionVMState", CallFlags.ReadStates, EmitLedgerGetTransactionVmStateArguments, dropResult: true), + CreateNativeMethod(NativeContract.ContractManagement, "getContract", CallFlags.ReadStates, EmitContractGetArguments), + CreateNativeMethod(NativeContract.ContractManagement, "getContractById", CallFlags.ReadStates, EmitContractGetByIdArguments), + CreateNativeMethod(NativeContract.ContractManagement, "getContractHashes", CallFlags.ReadStates, dropResult: true), + CreateNativeMethod(NativeContract.ContractManagement, "getMinimumDeploymentFee", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.ContractManagement, "hasMethod", CallFlags.ReadStates, EmitContractHasMethodArguments), + CreateNativeMethod(NativeContract.ContractManagement, "isContract", CallFlags.ReadStates, EmitContractIsContractArguments), + CreateNativeMethod( + NativeContract.ContractManagement, + "deploy", + CallFlags.States | CallFlags.AllowNotify, + EmitContractDeployArguments, + dropResult: true, + engineFactory: CreateContractDeploymentEngine, + profileFactory: _ => new ScenarioProfile(1, s_deployContractArtifact.NefBytes.Length + s_deployContractArtifact.ManifestBytes.Length, 1), + scriptFactory: (method, profile) => CreateSingleCallScriptSet(method, profile, s_setCallFlagsAll)), + CreateNativeMethod( + NativeContract.ContractManagement, + "update", + CallFlags.States | CallFlags.AllowNotify, + EmitContractUpdateArguments, + dropResult: false, + configureEngine: ConfigureContractUpdateEngine, + profileFactory: _ => new ScenarioProfile(1, s_updateTargetNefBytes.Length + s_updateTargetManifestBytes.Length, 1), + scriptFactory: (method, profile) => CreateContractSelfCallScriptSet(method, profile, s_updateContractArtifact.State)), + CreateNativeMethod( + NativeContract.ContractManagement, + "destroy", + CallFlags.States | CallFlags.AllowNotify, + emitArguments: null, + dropResult: false, + configureEngine: ConfigureContractDestroyEngine, + profileFactory: _ => new ScenarioProfile(1, 0, 0), + scriptFactory: (method, profile) => CreateContractSelfCallScriptSet(method, profile, s_destroyContractArtifact.State)), + CreateNativeMethod(NativeContract.ContractManagement, "setMinimumDeploymentFee", CallFlags.States, EmitContractSetMinimumDeploymentFeeArguments, dropResult: true, engineFactory: CreateCommitteeWitnessEngine), + CreateNativeMethod(NativeContract.Policy, "setFeePerByte", CallFlags.All, EmitPolicySetFeePerByteArguments, dropResult: true, engineFactory: CreateCommitteeWitnessEngine), + CreateNativeMethod(NativeContract.Policy, "setExecFeeFactor", CallFlags.All, EmitPolicySetExecFeeFactorArguments, dropResult: true, engineFactory: CreateCommitteeWitnessEngine), + CreateNativeMethod(NativeContract.Policy, "setStoragePrice", CallFlags.All, EmitPolicySetStoragePriceArguments, dropResult: true, engineFactory: CreateCommitteeWitnessEngine), + CreateNativeMethod(NativeContract.Policy, "setMillisecondsPerBlock", CallFlags.All, EmitPolicySetMillisecondsPerBlockArguments, dropResult: true, engineFactory: CreateCommitteeWitnessEngine), + CreateNativeMethod(NativeContract.Policy, "setMaxValidUntilBlockIncrement", CallFlags.All, EmitPolicySetMaxValidUntilBlockIncrementArguments, dropResult: true, engineFactory: CreateCommitteeWitnessEngine), + CreateNativeMethod(NativeContract.Policy, "setMaxTraceableBlocks", CallFlags.All, EmitPolicySetMaxTraceableBlocksArguments, dropResult: true, engineFactory: CreateCommitteeWitnessEngine), + CreateNativeMethod(NativeContract.Policy, "setAttributeFee", CallFlags.All, EmitPolicySetAttributeFeeArguments, dropResult: true, engineFactory: CreateCommitteeWitnessEngine), + CreateNativeMethod(NativeContract.Policy, "blockAccount", CallFlags.States, EmitPolicyBlockAccountArguments, dropResult: true, engineFactory: CreateCommitteeWitnessEngine), + CreateNativeMethod(NativeContract.Policy, "unblockAccount", CallFlags.States, EmitPolicyBlockAccountArguments, dropResult: true, engineFactory: CreateCommitteeWitnessEngine, configureEngine: ConfigurePolicyUnblockEngine), + CreateNativeMethod(NativeContract.NEO, "totalSupply", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.NEO, "balanceOf", CallFlags.ReadStates, EmitCommitteeAccountArguments), + CreateNativeMethod(NativeContract.NEO, "decimals", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.NEO, "symbol", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.NEO, "getAccountState", CallFlags.ReadStates, EmitNeoAccountStateArguments), + CreateNativeMethod(NativeContract.NEO, "getCandidateVote", CallFlags.ReadStates, EmitNeoCandidateVoteArguments), + CreateNativeMethod(NativeContract.NEO, "getCandidates", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.NEO, "getAllCandidates", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.NEO, "getCommittee", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.NEO, "getCommitteeAddress", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.NEO, "getGasPerBlock", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.NEO, "getRegisterPrice", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.NEO, "getNextBlockValidators", CallFlags.ReadStates, EmitNeoNextBlockValidatorsArguments), + CreateNativeMethod(NativeContract.NEO, "unclaimedGas", CallFlags.ReadStates, EmitNeoUnclaimedGasArguments), + CreateNativeMethod( + NativeContract.NEO, + "onNEP17Payment", + CallFlags.States | CallFlags.AllowNotify, + EmitNeoOnPaymentArguments, + dropResult: false, + engineFactory: CreateNeoOnPaymentEngine, + configureEngine: ConfigureNeoOnPaymentEngine, + profileFactory: _ => new ScenarioProfile(1, 96, 1), + scriptFactory: (method, profile) => CreateNeoScriptSet(method, profile, state => SetNativeCallingScriptHash(state, NativeContract.GAS.Hash))), + CreateNativeMethod( + NativeContract.NEO, + "registerCandidate", + CallFlags.States | CallFlags.AllowNotify, + EmitNeoRegisterCandidateArguments, + dropResult: false, + engineFactory: CreateNeoCandidateEngine, + configureEngine: ConfigureNeoRegisterCandidateEngine, + profileFactory: _ => new ScenarioProfile(1, 64, 1), + scriptFactory: (method, profile) => CreateNeoScriptSet(method, profile)), + CreateNativeMethod( + NativeContract.NEO, + "setGasPerBlock", + CallFlags.States, + EmitNeoSetGasPerBlockArguments, + dropResult: true, + engineFactory: CreateCommitteeWitnessEngine, + configureEngine: ConfigureNeoSetGasPerBlockEngine, + profileFactory: _ => new ScenarioProfile(1, 16, 1), + scriptFactory: (method, profile) => CreateNeoScriptSet(method, profile)), + CreateNativeMethod( + NativeContract.NEO, + "setRegisterPrice", + CallFlags.States, + EmitNeoSetRegisterPriceArguments, + dropResult: true, + engineFactory: CreateCommitteeWitnessEngine, + configureEngine: ConfigureNeoSetRegisterPriceEngine, + profileFactory: _ => new ScenarioProfile(1, 16, 1), + scriptFactory: (method, profile) => CreateNeoScriptSet(method, profile)), + CreateNativeMethod( + NativeContract.NEO, + "transfer", + CallFlags.All, + EmitNeoTransferArguments, + dropResult: false, + engineFactory: CreateNeoTransferEngine, + configureEngine: ConfigureNeoTransferEngine, + profileFactory: _ => new ScenarioProfile(1, 128, 4), + scriptFactory: (method, profile) => CreateNeoScriptSet(method, profile)), + CreateNativeMethod( + NativeContract.NEO, + "unregisterCandidate", + CallFlags.States | CallFlags.AllowNotify, + EmitNeoUnregisterCandidateArguments, + dropResult: false, + engineFactory: CreateNeoCandidateEngine, + configureEngine: ConfigureNeoUnregisterCandidateEngine, + profileFactory: _ => new ScenarioProfile(1, 64, 1), + scriptFactory: (method, profile) => CreateNeoScriptSet(method, profile)), + CreateNativeMethod( + NativeContract.NEO, + "vote", + CallFlags.States | CallFlags.AllowNotify, + EmitNeoVoteArguments, + dropResult: false, + engineFactory: CreateNeoVoteEngine, + configureEngine: ConfigureNeoVoteEngine, + profileFactory: _ => new ScenarioProfile(1, 96, 1), + scriptFactory: (method, profile) => CreateNeoScriptSet(method, profile)), + CreateNativeMethod(NativeContract.GAS, "totalSupply", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.GAS, "balanceOf", CallFlags.ReadStates, EmitCommitteeAccountArguments), + CreateNativeMethod(NativeContract.GAS, "decimals", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.GAS, "symbol", CallFlags.ReadStates), + CreateNativeMethod(NativeContract.GAS, "transfer", CallFlags.All, EmitGasTransferArguments, dropResult: true, engineFactory: CreateGasTransferEngine), + CreateNativeMethod(NativeContract.Notary, "balanceOf", CallFlags.ReadStates, EmitNotaryBalanceArguments, dropResult: true, engineFactory: _ => NativeBenchmarkStateFactory.CreateEngine(), configureEngine: ConfigureNotaryBalanceEngine, profileFactory: _ => new ScenarioProfile(1, 32, 1), scriptFactory: (method, profile) => CreateNotaryScriptSet(method, profile)), + CreateNativeMethod(NativeContract.Notary, "expirationOf", CallFlags.ReadStates, EmitNotaryExpirationArguments, dropResult: true, engineFactory: _ => NativeBenchmarkStateFactory.CreateEngine(), configureEngine: ConfigureNotaryExpirationEngine, profileFactory: _ => new ScenarioProfile(1, 32, 1), scriptFactory: (method, profile) => CreateNotaryScriptSet(method, profile)), + CreateNativeMethod(NativeContract.Notary, "getMaxNotValidBeforeDelta", CallFlags.ReadStates, dropResult: true, engineFactory: _ => NativeBenchmarkStateFactory.CreateEngine(), configureEngine: ConfigureNotarySetMaxDeltaEngine, profileFactory: _ => new ScenarioProfile(1, 0, 0), scriptFactory: (method, profile) => CreateNotaryScriptSet(method, profile)), + CreateNativeMethod(NativeContract.Notary, "lockDepositUntil", CallFlags.States, EmitNotaryLockDepositUntilArguments, dropResult: true, engineFactory: CreateNotaryAccountEngine, configureEngine: ConfigureNotaryLockDepositEngine, profileFactory: _ => new ScenarioProfile(1, 48, 1), scriptFactory: (method, profile) => CreateNotaryScriptSet(method, profile)), + CreateNativeMethod(NativeContract.Notary, "onNEP17Payment", CallFlags.States, EmitNotaryOnPaymentArguments, dropResult: true, engineFactory: CreateNotaryOnPaymentEngine, configureEngine: ConfigureNotaryOnPaymentEngine, profileFactory: _ => new ScenarioProfile(1, 96, 1), scriptFactory: (method, profile) => CreateNotaryScriptSet(method, profile, state => SetNativeCallingScriptHash(state, NativeContract.GAS.Hash))), + CreateNativeMethod(NativeContract.Notary, "setMaxNotValidBeforeDelta", CallFlags.States, EmitNotarySetMaxDeltaArguments, dropResult: true, engineFactory: CreateCommitteeWitnessEngine, configureEngine: ConfigureNotarySetMaxDeltaEngine, profileFactory: _ => new ScenarioProfile(1, 16, 1), scriptFactory: (method, profile) => CreateNotaryScriptSet(method, profile)), + CreateNativeMethod(NativeContract.Notary, "verify", CallFlags.ReadStates, EmitNotaryVerifyArguments, dropResult: true, engineFactory: CreateNotaryVerifyEngine, configureEngine: ConfigureNotaryVerifyEngine, profileFactory: _ => new ScenarioProfile(1, 64, 1), scriptFactory: (method, profile) => CreateNotaryScriptSet(method, profile)), + CreateNativeMethod(NativeContract.Notary, "withdraw", CallFlags.All, EmitNotaryWithdrawArguments, dropResult: true, engineFactory: CreateNotaryWithdrawEngine, configureEngine: ConfigureNotaryWithdrawEngine, profileFactory: _ => new ScenarioProfile(1, 64, 1), scriptFactory: (method, profile) => CreateNotaryScriptSet(method, profile)), + CreateNativeMethod(NativeContract.Oracle, "getPrice", CallFlags.ReadStates, dropResult: true, engineFactory: _ => NativeBenchmarkStateFactory.CreateEngine(), configureEngine: ConfigureOracleGetPriceEngine, profileFactory: _ => new ScenarioProfile(1, 0, 0), scriptFactory: (method, profile) => CreateOracleScriptSet(method, profile)), + CreateNativeMethod(NativeContract.Oracle, "setPrice", CallFlags.States, EmitOracleSetPriceArguments, dropResult: true, engineFactory: CreateCommitteeWitnessEngine, configureEngine: ConfigureOracleSetPriceEngine, profileFactory: _ => new ScenarioProfile(1, 16, 1), scriptFactory: (method, profile) => CreateOracleScriptSet(method, profile)), + CreateNativeMethod(NativeContract.Oracle, "request", CallFlags.States | CallFlags.AllowCall | CallFlags.AllowNotify, EmitOracleRequestArguments, dropResult: true, engineFactory: CreateOracleRequestEngine, configureEngine: ConfigureOracleRequestEngine, profileFactory: _ => new ScenarioProfile(1, 256, 1), scriptFactory: (method, profile) => CreateOracleScriptSet(method, profile)), + CreateNativeMethod(NativeContract.Oracle, "finish", CallFlags.States | CallFlags.AllowCall | CallFlags.AllowNotify, dropResult: true, engineFactory: CreateOracleFinishEngine, configureEngine: ConfigureOracleFinishEngine, profileFactory: _ => new ScenarioProfile(1, 0, 0), scriptFactory: (method, profile) => CreateOracleScriptSet(method, profile)), + CreateNativeMethod(NativeContract.Oracle, "verify", CallFlags.ReadStates, dropResult: true, engineFactory: CreateOracleVerifyEngine, configureEngine: ConfigureOracleVerifyEngine, profileFactory: _ => new ScenarioProfile(1, 0, 0), scriptFactory: (method, profile) => CreateOracleScriptSet(method, profile)), + CreateNativeMethod(NativeContract.CryptoLib, "keccak256", CallFlags.All, EmitCryptoHashArguments), + CreateNativeMethod(NativeContract.CryptoLib, "murmur32", CallFlags.All, EmitCryptoMurmurArguments), + CreateNativeMethod(NativeContract.CryptoLib, "ripemd160", CallFlags.All, EmitCryptoHashArguments), + CreateNativeMethod(NativeContract.CryptoLib, "sha256", CallFlags.All, EmitCryptoHashArguments), + CreateNativeMethod(NativeContract.CryptoLib, "bls12381Deserialize", CallFlags.All, EmitCryptoBlsDeserializeArguments), + CreateNativeMethod(NativeContract.CryptoLib, "bls12381Serialize", CallFlags.All, EmitCryptoBlsSerializeArguments), + CreateNativeMethod(NativeContract.CryptoLib, "bls12381Add", CallFlags.All, EmitCryptoBlsAddArguments), + CreateNativeMethod(NativeContract.CryptoLib, "bls12381Equal", CallFlags.All, EmitCryptoBlsEqualArguments), + CreateNativeMethod(NativeContract.CryptoLib, "bls12381Mul", CallFlags.All, EmitCryptoBlsMulArguments), + CreateNativeMethod(NativeContract.CryptoLib, "bls12381Pairing", CallFlags.All, EmitCryptoBlsPairingArguments), + CreateNativeMethod(NativeContract.CryptoLib, "recoverSecp256K1", CallFlags.All, EmitCryptoRecoverSecpArguments), + CreateNativeMethod(NativeContract.CryptoLib, "verifyWithECDsa", CallFlags.All, EmitCryptoVerifyEcdsaArguments), + CreateNativeMethod(NativeContract.CryptoLib, "verifyWithEd25519", CallFlags.All, EmitCryptoVerifyEd25519Arguments), + CreateNativeMethod(NativeContract.StdLib, "base58CheckDecode", CallFlags.All, EmitStdLibBase58CheckDecodeArguments), + CreateNativeMethod(NativeContract.StdLib, "base58CheckEncode", CallFlags.All, EmitStdLibBase58CheckEncodeArguments), + CreateNativeMethod(NativeContract.StdLib, "base58Decode", CallFlags.All, EmitStdLibBase58DecodeArguments), + CreateNativeMethod(NativeContract.StdLib, "base58Encode", CallFlags.All, EmitStdLibBase58EncodeArguments), + CreateNativeMethod(NativeContract.StdLib, "base64Decode", CallFlags.All, EmitStdLibBase64DecodeArguments), + CreateNativeMethod(NativeContract.StdLib, "base64Encode", CallFlags.All, EmitStdLibBase64EncodeArguments), + CreateNativeMethod(NativeContract.StdLib, "base64UrlDecode", CallFlags.All, EmitStdLibBase64UrlDecodeArguments), + CreateNativeMethod(NativeContract.StdLib, "base64UrlEncode", CallFlags.All, EmitStdLibBase64UrlEncodeArguments), + CreateNativeMethod(NativeContract.StdLib, "hexDecode", CallFlags.All, EmitStdLibHexDecodeArguments), + CreateNativeMethod(NativeContract.StdLib, "hexEncode", CallFlags.All, EmitStdLibHexEncodeArguments), + CreateNativeMethod(NativeContract.StdLib, "memoryCompare", CallFlags.All, EmitStdLibMemoryCompareArguments), + CreateNativeMethod(NativeContract.StdLib, "memorySearch", CallFlags.All, EmitStdLibMemorySearchArguments), + CreateNativeMethod(NativeContract.StdLib, "stringSplit", CallFlags.All, EmitStdLibStringSplitArguments), + CreateNativeMethod(NativeContract.StdLib, "atoi", CallFlags.All, EmitStdLibAtoiArguments), + CreateNativeMethod(NativeContract.StdLib, "itoa", CallFlags.All, EmitStdLibItoaArguments), + CreateNativeMethod(NativeContract.StdLib, "serialize", CallFlags.All, EmitStdLibSerializeArguments), + CreateNativeMethod(NativeContract.StdLib, "deserialize", CallFlags.All, EmitStdLibDeserializeArguments), + CreateNativeMethod(NativeContract.StdLib, "jsonSerialize", CallFlags.All, EmitStdLibJsonSerializeArguments), + CreateNativeMethod(NativeContract.StdLib, "jsonDeserialize", CallFlags.All, EmitStdLibJsonDeserializeArguments), + CreateNativeMethod(NativeContract.StdLib, "strLen", CallFlags.All, EmitStdLibStrLenArguments), + CreateNativeMethod(NativeContract.RoleManagement, "designateAsRole", CallFlags.States | CallFlags.AllowNotify, EmitRoleDesignateArguments, dropResult: true, engineFactory: CreateCommitteeWitnessEngine, configureEngine: ConfigureRoleDesignateEngine, profileFactory: _ => new ScenarioProfile(1, 96, 1), scriptFactory: (method, profile) => CreateRoleScriptSet(method, profile)), + CreateNativeMethod(NativeContract.RoleManagement, "getDesignatedByRole", CallFlags.ReadStates, EmitRoleGetDesignatedArguments, dropResult: true, engineFactory: _ => NativeBenchmarkStateFactory.CreateEngine(), configureEngine: ConfigureRoleGetDesignatedEngine, profileFactory: _ => new ScenarioProfile(1, 32, 1), scriptFactory: (method, profile) => CreateRoleScriptSet(method, profile)) + }; + + + public static IEnumerable CreateCases() + { + foreach (ScenarioComplexity complexity in Enum.GetValues()) + { + foreach (var method in Methods) + { + yield return BuildCase(method, complexity); + } + } + } + + private static VmBenchmarkCase BuildCase(NativeMethod method, ScenarioComplexity complexity) + { + var scenario = new ApplicationEngineVmScenario( + method.ScriptFactory is null + ? CreateScripts(method) + : profile => method.ScriptFactory(method, profile), + method.ConfigureEngine, + profile => method.EngineFactory?.Invoke(profile) ?? NativeBenchmarkStateFactory.CreateEngine()); + + var baseProfile = ScenarioProfile.For(complexity); + var effectiveProfile = method.ProfileFactory?.Invoke(baseProfile) ?? baseProfile; + + NativeCoverageTracker.Register(method.Id); + return new VmBenchmarkCase(method.Id, BenchmarkComponent.NativeContract, complexity, scenario, effectiveProfile); + } + + private static Func CreateScripts(NativeMethod method) + { + return profile => + { + var baselineProfile = profile.With(dataLength: 0, collectionLength: 0); + var baselineScript = CreateScript( + LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.AddInstruction(VM.OpCode.PUSH0); + builder.AddInstruction(VM.OpCode.DROP); + }), + baselineProfile); + + var singleScript = CreateScript(BuildCallLoop(method, profile), profile); + var saturatedProfile = new ScenarioProfile(profile.Iterations * 4, profile.DataLength * 2, profile.CollectionLength * 2); + var saturatedScript = CreateScript(BuildCallLoop(method, saturatedProfile), saturatedProfile); + + return new ApplicationEngineVmScenario.ApplicationEngineScriptSet(baselineScript, singleScript, saturatedScript); + }; + } + + private static ApplicationEngineVmScenario.ApplicationEngineScriptSet CreateSingleCallScriptSet( + NativeMethod method, + ScenarioProfile profile, + Action? configureState = null) + { + var iterations = Math.Max(1, profile.Iterations); + var baseline = CreateScript(BuildNoOpScriptBytes(), profile.With(iterations: iterations, dataLength: 0, collectionLength: 0), configureState); + var singleProfile = profile.With(iterations: iterations); + var single = CreateScript(BuildSingleCallScriptBytes(method, singleProfile), singleProfile, configureState); + return new ApplicationEngineVmScenario.ApplicationEngineScriptSet(baseline, single, single); + } + + private static ApplicationEngineVmScenario.ApplicationEngineScriptSet CreateNeoScriptSet( + NativeMethod method, + ScenarioProfile profile, + Action? additionalConfiguration = null) + { + return CreateSingleCallScriptSet(method, profile, state => + { + additionalConfiguration?.Invoke(state); + state.ScriptHash = NativeContract.NEO.Hash; + state.Contract = CloneContractState(NativeContract.NEO.GetContractState(ProtocolSettings.Default, 0)); + }); + } + + private static ApplicationEngineVmScenario.ApplicationEngineScriptSet CreateContractSelfCallScriptSet( + NativeMethod method, + ScenarioProfile profile, + ContractState contract, + Action? additionalConfiguration = null) + { + var configure = CombineConfigurations(additionalConfiguration, state => + { + state.ScriptHash = contract.Hash; + state.Contract = CloneContractState(contract); + state.CallFlags = CallFlags.All; + }); + + return CreateSingleCallScriptSet(method, profile, configure); + } + + private static Action? CombineConfigurations(params Action?[] configurations) + { + if (configurations.All(action => action is null)) + return null; + + return state => + { + foreach (var action in configurations) + action?.Invoke(state); + }; + } + + private static ApplicationEngineVmScenario.ApplicationEngineScript CreateScript( + byte[] script, + ScenarioProfile profile, + Action? configureState = null) + { + return new ApplicationEngineVmScenario.ApplicationEngineScript(script, profile, configureState); + } + + private static void SetNativeCallingScriptHash(ExecutionContextState state, UInt160 hash) + { + s_nativeCallingScriptHashProperty?.SetValue(state, hash); + } + + private static byte[] BuildSingleCallScriptBytes(NativeMethod method, ScenarioProfile profile) + { + var builder = new InstructionBuilder(); + if (method.EmitArguments is null) + { + builder.AddInstruction(VM.OpCode.NEWARRAY0); + } + else + { + method.EmitArguments(builder, profile); + } + + builder.Push((int)method.Flags); + builder.Push(method.MethodName); + builder.Push(method.ContractHash.GetSpan().ToArray()); + builder.AddInstruction(new Instruction + { + _opCode = VM.OpCode.SYSCALL, + _operand = BitConverter.GetBytes(ApplicationEngine.System_Contract_Call.Hash) + }); + + if (method.DropResult) + { + builder.AddInstruction(VM.OpCode.DROP); + } + + builder.AddInstruction(VM.OpCode.RET); + return builder.ToArray(); + } + + private static byte[] BuildNoOpScriptBytes() + { + var builder = new InstructionBuilder(); + builder.AddInstruction(VM.OpCode.RET); + return builder.ToArray(); + } + + private static BenchmarkApplicationEngine CreateNeoOnPaymentEngine(ScenarioProfile profile) + { + var transaction = new Transaction + { + Signers = new[] + { + new Signer + { + Account = s_candidateAccount, + Scopes = WitnessScope.Global + } + }, + Attributes = System.Array.Empty(), + Witnesses = System.Array.Empty(), + Script = System.Array.Empty() + }; + + return NativeBenchmarkStateFactory.CreateEngine(transaction); + } + + private static BenchmarkApplicationEngine CreateNeoCandidateEngine(ScenarioProfile profile) + { + return NativeBenchmarkStateFactory.CreateEngine(new ManualWitness(s_candidateAccount)); + } + + private static BenchmarkApplicationEngine CreateNeoTransferEngine(ScenarioProfile profile) + { + return NativeBenchmarkStateFactory.CreateEngine(new ManualWitness(s_neoSenderAccount)); + } + + private static BenchmarkApplicationEngine CreateNeoVoteEngine(ScenarioProfile profile) + { + return NativeBenchmarkStateFactory.CreateEngine(new ManualWitness(s_neoSenderAccount)); + } + + private static void ConfigureNeoOnPaymentEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.NEO); + SeedNeoRegisterPrice(engine.SnapshotCache, s_defaultRegisterPrice); + SeedNeoAccount(engine.SnapshotCache, s_candidateAccount, 0); + SeedNeoCandidate(engine.SnapshotCache, s_candidatePublicKey, registered: false, votes: BigInteger.Zero); + var depositAmount = new BigInteger(PolicyContract.DefaultNotaryAssistedAttributeFee) * 2; + SeedGasAccount(engine.SnapshotCache, NativeContract.NEO.Hash, depositAmount); + } + + private static void ConfigureNeoRegisterCandidateEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.NEO); + SeedNeoRegisterPrice(engine.SnapshotCache, s_defaultRegisterPrice); + SeedNeoCandidate(engine.SnapshotCache, s_candidatePublicKey, registered: false, votes: BigInteger.Zero); + SeedNeoAccount(engine.SnapshotCache, s_candidateAccount, NativeContract.NEO.Factor); + } + + private static void ConfigureNeoSetGasPerBlockEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.NEO); + SeedNeoRegisterPrice(engine.SnapshotCache, s_defaultRegisterPrice); + } + + private static void ConfigureNeoSetRegisterPriceEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.NEO); + SeedNeoRegisterPrice(engine.SnapshotCache, s_defaultRegisterPrice); + } + + private static void ConfigureNeoTransferEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.NEO); + var balance = 10 * NativeContract.NEO.Factor; + SeedNeoAccount(engine.SnapshotCache, s_neoSenderAccount, balance); + SeedNeoAccount(engine.SnapshotCache, s_neoRecipientAccount, 0); + } + + private static void ConfigureNeoUnregisterCandidateEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.NEO); + SeedNeoRegisterPrice(engine.SnapshotCache, s_defaultRegisterPrice); + SeedNeoCandidate(engine.SnapshotCache, s_candidatePublicKey, registered: true, votes: BigInteger.Zero); + } + + private static void ConfigureNeoVoteEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.NEO); + SeedNeoRegisterPrice(engine.SnapshotCache, s_defaultRegisterPrice); + SeedNeoVotersCount(engine.SnapshotCache); + SeedNeoCandidate(engine.SnapshotCache, s_candidatePublicKey, registered: true, votes: BigInteger.Zero); + SeedNeoAccount(engine.SnapshotCache, s_neoSenderAccount, 5 * NativeContract.NEO.Factor, balanceHeight: 0, voteTo: null); + } + + private static ApplicationEngineVmScenario.ApplicationEngineScriptSet CreateNotaryScriptSet( + NativeMethod method, + ScenarioProfile profile, + Action? additionalConfiguration = null) + { + return CreateSingleCallScriptSet(method, profile, state => + { + additionalConfiguration?.Invoke(state); + state.ScriptHash = NativeContract.Notary.Hash; + state.Contract = CloneContractState(NativeContract.Notary.GetContractState(ProtocolSettings.Default, 0)); + }); + } + + private static BenchmarkApplicationEngine CreateNotaryOnPaymentEngine(ScenarioProfile profile) + { + var transaction = new Transaction + { + Signers = new[] + { + new Signer + { + Account = s_notaryAccount, + Scopes = WitnessScope.Global + } + }, + Attributes = System.Array.Empty(), + Witnesses = System.Array.Empty(), + Script = System.Array.Empty() + }; + + return NativeBenchmarkStateFactory.CreateEngine(transaction); + } + + private static BenchmarkApplicationEngine CreateNotaryAccountEngine(ScenarioProfile profile) + { + return NativeBenchmarkStateFactory.CreateEngine(new ManualWitness(s_notaryAccount)); + } + + private static BenchmarkApplicationEngine CreateNotaryVerifyEngine(ScenarioProfile profile) + { + var transaction = new Transaction + { + Signers = new[] { new Signer { Account = s_notaryAccount, Scopes = WitnessScope.Global } }, + Attributes = new TransactionAttribute[] + { + new NotaryAssisted { NKeys = 0 } + }, + Witnesses = System.Array.Empty(), + Script = System.Array.Empty() + }; + + return NativeBenchmarkStateFactory.CreateEngine(transaction); + } + + private static BenchmarkApplicationEngine CreateNotaryWithdrawEngine(ScenarioProfile profile) + { + return NativeBenchmarkStateFactory.CreateEngine(new ManualWitness(s_notaryAccount)); + } + + private static void ConfigureNotaryBalanceEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.Notary); + SeedNotaryDeposit(engine.SnapshotCache, s_notaryAccount, new BigInteger(500_00000000), s_notaryInitialTill); + } + + private static void ConfigureNotaryExpirationEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.Notary); + SeedNotaryDeposit(engine.SnapshotCache, s_notaryAccount, new BigInteger(500_00000000), s_notaryInitialTill); + } + + private static void ConfigureNotarySetMaxDeltaEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.Notary); + SeedNotaryMaxDelta(engine.SnapshotCache, NotaryDefaultMaxNotValidBeforeDelta); + } + + private static void ConfigureNotaryLockDepositEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.Notary); + SeedNotaryDeposit(engine.SnapshotCache, s_notaryAccount, new BigInteger(500_00000000), s_notaryInitialTill); + } + + private static void ConfigureNotaryOnPaymentEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.Notary); + SeedNotaryMaxDelta(engine.SnapshotCache, NotaryDefaultMaxNotValidBeforeDelta); + ClearNotaryDeposit(engine.SnapshotCache, s_notaryAccount); + } + + private static void ConfigureNotaryVerifyEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.Notary); + SeedNotaryMaxDelta(engine.SnapshotCache, NotaryDefaultMaxNotValidBeforeDelta); + } + + private static void ConfigureNotaryWithdrawEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.Notary); + SeedNotaryDeposit(engine.SnapshotCache, s_notaryAccount, new BigInteger(800_00000000), till: 0); + SeedGasAccount(engine.SnapshotCache, NativeContract.Notary.Hash, new BigInteger(800_00000000)); + } + + private static void SeedNotaryDeposit(DataCache snapshot, UInt160 account, BigInteger amount, uint till) + { + var key = StorageKey.Create(NativeContract.Notary.Id, NotaryDepositPrefix, account); + snapshot.Delete(key); + var deposit = new Notary.Deposit + { + Amount = amount, + Till = till + }; + snapshot.Add(key, new StorageItem(deposit)); + } + + private static void SeedNotaryMaxDelta(DataCache snapshot, uint value) + { + var key = StorageKey.Create(NativeContract.Notary.Id, NotaryMaxNotValidBeforeDeltaPrefix); + var item = snapshot.GetAndChange(key, () => new StorageItem(BigInteger.Zero)); + item.Set(value); + } + + private static void ClearNotaryDeposit(DataCache snapshot, UInt160 account) + { + var key = StorageKey.Create(NativeContract.Notary.Id, NotaryDepositPrefix, account); + snapshot.Delete(key); + } + + private static ApplicationEngineVmScenario.ApplicationEngineScriptSet CreateOracleScriptSet( + NativeMethod method, + ScenarioProfile profile, + Action? additionalConfiguration = null) + { + return CreateSingleCallScriptSet(method, profile, state => + { + additionalConfiguration?.Invoke(state); + state.ScriptHash = NativeContract.Oracle.Hash; + state.Contract = CloneContractState(NativeContract.Oracle.GetContractState(ProtocolSettings.Default, 0)); + }); + } + + private static BenchmarkApplicationEngine CreateOracleRequestEngine(ScenarioProfile profile) + { + var transaction = CreateOracleRequestTransaction(s_oracleCallbackContract.State.Hash); + return NativeBenchmarkStateFactory.CreateEngine(transaction); + } + + private static BenchmarkApplicationEngine CreateOracleFinishEngine(ScenarioProfile profile) + { + var response = new OracleResponse + { + Id = s_oracleRequestId, + Code = OracleResponseCode.Success, + Result = s_oracleSampleResult + }; + var tx = CreateOracleResponseTransaction(response); + return NativeBenchmarkStateFactory.CreateEngine(tx); + } + + private static BenchmarkApplicationEngine CreateOracleVerifyEngine(ScenarioProfile profile) + { + var response = new OracleResponse + { + Id = s_oracleRequestId, + Code = OracleResponseCode.Success, + Result = s_oracleSampleResult + }; + var tx = CreateOracleResponseTransaction(response); + return NativeBenchmarkStateFactory.CreateEngine(tx); + } + + private static void ConfigureOracleGetPriceEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.Oracle); + SeedOraclePrice(engine.SnapshotCache, 500_000000); + } + + private static void ConfigureOracleSetPriceEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.Oracle); + SeedOraclePrice(engine.SnapshotCache, 500_000000); + } + + private static void ConfigureOracleRequestEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.Oracle); + EnsureContract(engine, s_oracleCallbackContract.State); + SeedOraclePrice(engine.SnapshotCache, 500_000000); + SeedOracleRequestId(engine.SnapshotCache, s_oracleRequestId); + } + + private static void ConfigureOracleFinishEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.Oracle); + EnsureContract(engine, s_oracleCallbackContract.State); + SeedOraclePrice(engine.SnapshotCache, 500_000000); + SeedOracleRequest(engine.SnapshotCache, s_oracleRequestId, CreateOracleRequestPayload()); + SeedOracleIdList(engine.SnapshotCache, s_oracleRequestId, OracleSampleUrl); + SeedOracleRequestId(engine.SnapshotCache, s_oracleRequestId + 1); + } + + private static void ConfigureOracleVerifyEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.Oracle); + ConfigureOracleFinishEngine(engine, profile); + } + + private static void SeedOraclePrice(DataCache snapshot, long price) + { + var key = StorageKey.Create(NativeContract.Oracle.Id, OraclePricePrefix); + var item = snapshot.GetAndChange(key, () => new StorageItem(BigInteger.Zero)); + item.Set(price); + } + + private static void SeedOracleRequestId(DataCache snapshot, ulong nextId) + { + var key = StorageKey.Create(NativeContract.Oracle.Id, OracleRequestIdPrefix); + var item = snapshot.GetAndChange(key, () => new StorageItem(BigInteger.Zero)); + item.Set(new BigInteger(nextId)); + } + + private static void SeedOracleRequest(DataCache snapshot, ulong id, OracleRequest request) + { + var key = StorageKey.Create(NativeContract.Oracle.Id, OracleRequestPrefix, (long)id); + snapshot.Delete(key); + snapshot.Add(key, StorageItem.CreateSealed(request)); + } + + private static void SeedOracleIdList(DataCache snapshot, ulong id, string url) + { + var hash = Crypto.Hash160(url.ToStrictUtf8Bytes()); + var key = StorageKey.Create(NativeContract.Oracle.Id, OracleIdListPrefix, hash.AsSpan()); + snapshot.Delete(key); + snapshot.Add(key, new StorageItem(CreateOracleIdList(ids: new[] { id }))); + } + + private static OracleRequest CreateOracleRequestPayload() + { + return new OracleRequest + { + OriginalTxid = UInt256.Zero, + GasForResponse = 200_0000000, + Url = OracleSampleUrl, + Filter = OracleSampleFilter, + CallbackContract = s_oracleCallbackContract.State.Hash, + CallbackMethod = OracleSampleCallback, + UserData = BinarySerializer.Serialize(VMStackItem.Null, OracleMaxUserDataLength, ExecutionEngineLimits.Default.MaxStackSize) + }; + } + + private static Transaction CreateOracleRequestTransaction(UInt160 caller) + { + return new Transaction + { + Signers = new[] { new Signer { Account = caller, Scopes = WitnessScope.Global } }, + Attributes = System.Array.Empty(), + Witnesses = System.Array.Empty(), + Script = System.Array.Empty() + }; + } + + private static Transaction CreateOracleResponseTransaction(OracleResponse response) + { + return new Transaction + { + Signers = new[] { new Signer { Account = s_oracleCallbackContract.State.Hash, Scopes = WitnessScope.Global } }, + Attributes = new TransactionAttribute[] { response }, + Witnesses = System.Array.Empty(), + Script = OracleResponse.FixedScript, + NetworkFee = 200_0000000L, + SystemFee = 0 + }; + } + + private static IInteroperable CreateOracleIdList(IEnumerable ids) + { + if (s_oracleIdListType is null) + throw new InvalidOperationException("Oracle IdList type unavailable."); + var instance = Activator.CreateInstance(s_oracleIdListType); + if (instance is not IInteroperable interoperable) + throw new InvalidOperationException("Oracle IdList does not implement IInteroperable."); + s_idListAddRangeMethod?.Invoke(instance, new object[] { ids }); + return interoperable; + } + + private static ApplicationEngineVmScenario.ApplicationEngineScriptSet CreateRoleScriptSet( + NativeMethod method, + ScenarioProfile profile, + Action? additionalConfiguration = null) + { + return CreateSingleCallScriptSet(method, profile, state => + { + additionalConfiguration?.Invoke(state); + state.ScriptHash = NativeContract.RoleManagement.Hash; + state.Contract = CloneContractState(NativeContract.RoleManagement.GetContractState(ProtocolSettings.Default, 0)); + }); + } + + private static void ConfigureRoleDesignateEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.RoleManagement); + ClearRoleDesignation(engine.SnapshotCache, Role.Oracle, engine.PersistingBlock.Index + 1); + } + + private static void ConfigureRoleGetDesignatedEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureNativeContract(engine, NativeContract.RoleManagement); + SeedRoleDesignation(engine.SnapshotCache, Role.Oracle, s_roleDesignationIndex, s_roleDesignationNodes); + } + + private static void SeedRoleDesignation(DataCache snapshot, Role role, uint index, IEnumerable nodes) + { + if (s_roleNodeListType is null) + throw new InvalidOperationException("Role node list type unavailable."); + var instance = Activator.CreateInstance(s_roleNodeListType); + if (instance is not IInteroperable interoperable) + throw new InvalidOperationException("NodeList does not implement IInteroperable."); + s_nodeListAddRangeMethod?.Invoke(instance, new object[] { nodes }); + s_nodeListSortMethod?.Invoke(instance, System.Array.Empty()); + + var key = StorageKey.Create(NativeContract.RoleManagement.Id, (byte)role, index); + snapshot.Delete(key); + snapshot.Add(key, new StorageItem(interoperable)); + } + + private static void ClearRoleDesignation(DataCache snapshot, Role role, uint index) + { + var key = StorageKey.Create(NativeContract.RoleManagement.Id, (byte)role, index); + snapshot.Delete(key); + } + + private static void EnsureNativeContract(BenchmarkApplicationEngine engine, NativeContract contract) + { + var contractState = CloneContractState(contract.GetContractState(ProtocolSettings.Default, 0)); + EnsureContract(engine, contractState); + } + + private static void SeedNeoRegisterPrice(DataCache snapshot, BigInteger price) + { + var key = StorageKey.Create(NativeContract.NEO.Id, NeoRegisterPricePrefix); + var item = snapshot.GetAndChange(key, () => new StorageItem(BigInteger.Zero)); + item.Set(price); + } + + private static void SeedNeoAccount(DataCache snapshot, UInt160 account, BigInteger balance, uint balanceHeight = 0, ECPoint? voteTo = null, BigInteger? lastGasPerVote = null) + { + var key = StorageKey.Create(NativeContract.NEO.Id, NeoAccountPrefix, account); + var item = snapshot.GetAndChange(key, () => new StorageItem(new NeoToken.NeoAccountState())); + var state = item.GetInteroperable(); + state.Balance = balance; + state.BalanceHeight = balanceHeight; + state.VoteTo = voteTo; + state.LastGasPerVote = lastGasPerVote ?? BigInteger.Zero; + } + + private static void SeedNeoCandidate(DataCache snapshot, ECPoint pubKey, bool registered, BigInteger votes) + { + if (s_candidateStateType is null || s_candidateRegisteredField is null || s_candidateVotesField is null) + return; + + var key = StorageKey.Create(NativeContract.NEO.Id, NeoCandidatePrefix, pubKey); + var item = snapshot.GetAndChange(key, () => new StorageItem(CreateCandidateState(registered, votes))); + if (s_storageItemGetInteroperable is null) + return; + + var state = s_storageItemGetInteroperable.MakeGenericMethod(s_candidateStateType) + .Invoke(item, System.Array.Empty()); + if (state is null) + return; + s_candidateRegisteredField.SetValue(state, registered); + s_candidateVotesField.SetValue(state, votes); + } + + private static void SeedNeoVotersCount(DataCache snapshot) + { + var key = StorageKey.Create(NativeContract.NEO.Id, NeoVotersCountPrefix); + if (!snapshot.Contains(key)) + snapshot.Add(key, new StorageItem(BigInteger.Zero)); + } + + private static IInteroperable CreateCandidateState(bool registered, BigInteger votes) + { + if (s_candidateStateType is null) + throw new InvalidOperationException("Candidate state type is unavailable."); + var instance = Activator.CreateInstance(s_candidateStateType); + if (instance is not IInteroperable interoperable) + throw new InvalidOperationException("Candidate state does not implement IInteroperable."); + s_candidateRegisteredField?.SetValue(instance, registered); + s_candidateVotesField?.SetValue(instance, votes); + return interoperable; + } + + + private static byte[] BuildCallLoop(NativeMethod method, ScenarioProfile profile) + { + return LoopScriptFactory.BuildCountingLoop(profile, builder => + { + if (method.EmitArguments is null) + { + builder.AddInstruction(VM.OpCode.NEWARRAY0); + } + else + { + method.EmitArguments(builder, profile); + } + + builder.Push((int)method.Flags); + builder.Push(method.MethodName); + builder.Push(method.ContractHash.GetSpan().ToArray()); + builder.AddInstruction(new Instruction + { + _opCode = VM.OpCode.SYSCALL, + _operand = BitConverter.GetBytes(ApplicationEngine.System_Contract_Call.Hash) + }); + + if (method.DropResult) + { + builder.AddInstruction(VM.OpCode.DROP); + } + }); + } + + private static byte[] BuildSimpleScript(Action emit) + { + var builder = new InstructionBuilder(); + emit(builder); + builder.AddInstruction(VM.OpCode.RET); + return builder.ToArray(); + } + + private static byte[] BuildContractManagementInvocationScript( + string methodName, + CallFlags flags, + Action? emitArguments) + { + var builder = new InstructionBuilder(); + + if (emitArguments is null) + { + builder.AddInstruction(VM.OpCode.NEWARRAY0); + } + else + { + emitArguments(builder, ScenarioProfile.For(ScenarioComplexity.Standard)); + } + + builder.Push((int)flags); + builder.Push(methodName); + builder.Push(NativeContract.ContractManagement.Hash.GetSpan().ToArray()); + builder.AddInstruction(new Instruction + { + _opCode = VM.OpCode.SYSCALL, + _operand = BitConverter.GetBytes(ApplicationEngine.System_Contract_Call.Hash) + }); + builder.AddInstruction(VM.OpCode.RET); + return builder.ToArray(); + } + + private static (NefFile Nef, ContractManifest Manifest, byte[] NefBytes, byte[] ManifestBytes) CreateContractDocument(string name, byte[] script) + { + var nef = new NefFile + { + Compiler = "benchmark", + Source = string.Empty, + Tokens = System.Array.Empty(), + Script = script + }; + nef.CheckSum = NefFile.ComputeChecksum(nef); + + var methodDescriptor = new ContractMethodDescriptor + { + Name = "main", + Parameters = System.Array.Empty(), + ReturnType = ContractParameterType.Any, + Offset = 0, + Safe = true + }; + + var manifest = new ContractManifest + { + Name = name, + Groups = System.Array.Empty(), + SupportedStandards = System.Array.Empty(), + Abi = new ContractAbi + { + Methods = new[] { methodDescriptor }, + Events = System.Array.Empty() + }, + Permissions = new[] { ContractPermission.DefaultPermission }, + Trusts = WildcardContainer.CreateWildcard(), + Extra = new JObject() + }; + + var nefBytes = nef.ToArray(); + var manifestBytes = manifest.ToJson().ToString().ToStrictUtf8Bytes(); + + return (nef, manifest, nefBytes, manifestBytes); + } + + private static ContractArtifact CreateContractArtifact(string name, byte[] script, int id) + { + var document = CreateContractDocument(name, script); + var contract = new ContractState + { + Id = id, + UpdateCounter = 0, + Hash = global::Neo.SmartContract.Helper.GetContractHash(s_contractDeployer, document.Nef.CheckSum, document.Manifest.Name), + Nef = document.Nef, + Manifest = document.Manifest + }; + + return new ContractArtifact(contract, document.NefBytes, document.ManifestBytes); + } + + private static ContractState CloneContractState(ContractState contract) + { + return new ContractState + { + Id = contract.Id, + UpdateCounter = contract.UpdateCounter, + Hash = contract.Hash, + Nef = NefFile.Parse(contract.Nef.ToArray()), + Manifest = ContractManifest.Parse(contract.Manifest.ToJson().ToString()) + }; + } + + private static void EnsureContract(BenchmarkApplicationEngine engine, ContractState contract) + { + var contractKey = new KeyBuilder(NativeContract.ContractManagement.Id, ContractManagementStoragePrefixContract) + .Add(contract.Hash); + + if (engine.SnapshotCache.Contains(contractKey)) + return; + + var clone = CloneContractState(contract); + engine.SnapshotCache.Add(contractKey, StorageItem.CreateSealed(clone)); + + var idKey = new KeyBuilder(NativeContract.ContractManagement.Id, ContractManagementStoragePrefixContractHash) + .AddBigEndian(clone.Id); + + if (!engine.SnapshotCache.Contains(idKey)) + engine.SnapshotCache.Add(idKey, new StorageItem(contract.Hash.ToArray())); + } + + private static BenchmarkApplicationEngine CreateContractDeploymentEngine(ScenarioProfile profile) + { + var transaction = new Transaction + { + Signers = new[] { new Signer { Account = s_contractDeployer, Scopes = WitnessScope.Global } }, + Attributes = System.Array.Empty(), + Script = System.Array.Empty(), + Witnesses = System.Array.Empty() + }; + + return NativeBenchmarkStateFactory.CreateEngine(transaction); + } + + private static void ConfigureContractUpdateEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureContract(engine, s_updateContractArtifact.State); + } + + private static void ConfigureContractDestroyEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + EnsureContract(engine, s_destroyContractArtifact.State); + } + + private static void EmitContractDeployArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_deployContractArtifact.NefBytes); + builder.Push(s_deployContractArtifact.ManifestBytes); + builder.Push(2); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitContractUpdateArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_updateTargetNefBytes); + builder.Push(s_updateTargetManifestBytes); + builder.Push(2); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitContractGetArguments(InstructionBuilder builder, ScenarioProfile profile) + { + EmitPackedSingleArgument(builder, NativeContract.NEO.Hash.GetSpan().ToArray()); + } + + private static void EmitContractGetByIdArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(NativeContract.NEO.Id); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitContractHasMethodArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(NativeContract.NEO.Hash.GetSpan().ToArray()); + builder.Push("transfer"); + builder.Push(4); + builder.Push(3); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitNeoOnPaymentArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_candidateAccount.ToArray()); + builder.Push(s_defaultRegisterPrice); + builder.Push(s_candidatePublicKey.EncodePoint(true)); + builder.Push(3); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitNeoRegisterCandidateArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_candidatePublicKey.EncodePoint(true)); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitNeoSetGasPerBlockArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(6 * NativeContract.GAS.Factor); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitNeoSetRegisterPriceArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(1500 * NativeContract.GAS.Factor); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitNeoTransferArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_neoSenderAccount.ToArray()); + builder.Push(s_neoRecipientAccount.ToArray()); + builder.Push(NativeContract.NEO.Factor); + builder.AddInstruction(VM.OpCode.PUSHNULL); + builder.Push(4); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitNeoUnregisterCandidateArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_candidatePublicKey.EncodePoint(true)); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitNeoVoteArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_neoSenderAccount.ToArray()); + builder.Push(s_candidatePublicKey.EncodePoint(true)); + builder.Push(2); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitNotaryBalanceArguments(InstructionBuilder builder, ScenarioProfile profile) + { + EmitPackedSingleArgument(builder, s_notaryAccount.ToArray()); + } + + private static void EmitNotaryExpirationArguments(InstructionBuilder builder, ScenarioProfile profile) + { + EmitPackedSingleArgument(builder, s_notaryAccount.ToArray()); + } + + private static void EmitNotaryLockDepositUntilArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_notaryAccount.ToArray()); + builder.Push(s_notaryLockTargetTill); + builder.Push(2); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitNotaryOnPaymentArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_notaryAccount.ToArray()); + builder.Push(new BigInteger(PolicyContract.DefaultNotaryAssistedAttributeFee) * 2); + builder.AddInstruction(VM.OpCode.PUSHNULL); + builder.Push(s_notaryInitialTill); + builder.Push(2); + builder.AddInstruction(VM.OpCode.PACK); + builder.Push(3); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitNotarySetMaxDeltaArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push((uint)NotaryDefaultMaxNotValidBeforeDelta / 2); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitNotaryVerifyArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_notaryDummySignature); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitNotaryWithdrawArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_notaryAccount.ToArray()); + builder.AddInstruction(VM.OpCode.PUSHNULL); + builder.Push(2); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitOracleSetPriceArguments(InstructionBuilder builder, ScenarioProfile profile) + { + var basePrice = Math.Max(1, profile.DataLength) * 10_000L; + builder.Push(basePrice); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitOracleRequestArguments(InstructionBuilder builder, ScenarioProfile profile) + { + var path = BenchmarkDataFactory.CreateString(Math.Max(4, profile.DataLength / 8), 'u'); + builder.Push($"{OracleSampleUrl}/{path}"); + builder.Push(OracleSampleFilter); + builder.Push(OracleSampleCallback); + builder.Push(BenchmarkDataFactory.CreateByteArray(Math.Max(1, profile.DataLength), 0x90)); + var gas = Math.Max(200_0000L, (long)profile.DataLength * 1_0000L); + builder.Push(gas); + builder.Push(Math.Max(1, profile.CollectionLength)); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitRoleDesignateArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push((int)Role.Oracle); + foreach (var node in s_roleDesignationNodes) + { + builder.Push(node.EncodePoint(true)); + } + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + builder.Push(2); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitRoleGetDesignatedArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push((int)Role.Oracle); + builder.Push(s_roleDesignationIndex); + builder.Push(2); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitContractIsContractArguments(InstructionBuilder builder, ScenarioProfile profile) + { + EmitPackedSingleArgument(builder, NativeContract.NEO.Hash.GetSpan().ToArray()); + } + + private static void EmitContractSetMinimumDeploymentFeeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(10_00000000); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitPolicySetFeePerByteArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_policyFeePerByteValue); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitPolicySetExecFeeFactorArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_policyExecFeeFactorValue); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitPolicySetStoragePriceArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_policyStoragePriceValue); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitPolicySetMillisecondsPerBlockArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_policyMillisecondsPerBlockValue); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitPolicySetMaxValidUntilBlockIncrementArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_policyMaxValidUntilValue); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitPolicySetMaxTraceableBlocksArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_policyMaxTraceableValue); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitPolicySetAttributeFeeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_policyAttributeType); + builder.Push(s_policyAttributeFeeValue); + builder.Push(2); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitPolicyBlockAccountArguments(InstructionBuilder builder, ScenarioProfile profile) + { + EmitPackedSingleArgument(builder, s_gasTransferRecipient.GetSpan().ToArray()); + } + + private static void EmitLedgerGetBlockArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(0); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitLedgerGetTransactionArguments(InstructionBuilder builder, ScenarioProfile profile) + { + EmitPackedSingleArgument(builder, s_zeroHashBytes); + } + + private static void EmitLedgerGetTransactionFromBlockArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_blockIndexBytes); + builder.Push(0); + builder.Push(2); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitLedgerGetTransactionHeightArguments(InstructionBuilder builder, ScenarioProfile profile) + { + EmitPackedSingleArgument(builder, s_zeroHashBytes); + } + + private static void EmitLedgerGetTransactionSignersArguments(InstructionBuilder builder, ScenarioProfile profile) + { + EmitPackedSingleArgument(builder, s_zeroHashBytes); + } + + private static void EmitLedgerGetTransactionVmStateArguments(InstructionBuilder builder, ScenarioProfile profile) + { + EmitPackedSingleArgument(builder, s_zeroHashBytes); + } + + private static void EmitGasTransferArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.AddInstruction(VM.OpCode.PUSHNULL); + builder.Push(s_gasTransferAmount); + builder.Push(s_gasTransferRecipient.GetSpan().ToArray()); + builder.Push(CommitteeAddress.GetSpan().ToArray()); + builder.Push(4); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitNeoAccountStateArguments(InstructionBuilder builder, ScenarioProfile profile) + { + EmitPackedSingleArgument(builder, CommitteeAddress.GetSpan().ToArray()); + } + + private static void EmitNeoCandidateVoteArguments(InstructionBuilder builder, ScenarioProfile profile) + { + EmitPackedSingleArgument(builder, s_committeePublicKeyCompressed); + } + + private static void EmitNeoNextBlockValidatorsArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_validatorsCount); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitNeoUnclaimedGasArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(1); + builder.Push(CommitteeAddress.GetSpan().ToArray()); + builder.Push(2); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitCommitteeAccountArguments(InstructionBuilder builder, ScenarioProfile profile) + { + EmitPackedSingleArgument(builder, CommitteeAddress.GetSpan().ToArray()); + } + + private static void EmitPackedSingleArgument(InstructionBuilder builder, byte[] payload) + { + builder.Push(payload); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static BenchmarkApplicationEngine CreateCommitteeWitnessEngine(ScenarioProfile profile) + { + return NativeBenchmarkStateFactory.CreateEngine(new ManualWitness(CommitteeAddress)); + } + + private static BenchmarkApplicationEngine CreateGasTransferEngine(ScenarioProfile profile) + { + var engine = NativeBenchmarkStateFactory.CreateEngine(new ManualWitness(CommitteeAddress)); + var iterations = Math.Max(1, profile.Iterations * 8); + var balance = s_gasTransferAmount * iterations; + SeedGasAccount(engine.SnapshotCache, CommitteeAddress, balance); + SeedGasAccount(engine.SnapshotCache, s_gasTransferRecipient, BigInteger.Zero); + return engine; + } + + private static void ConfigurePolicyUnblockEngine(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + var key = new KeyBuilder(NativeContract.Policy.Id, PolicyBlockedAccountPrefix).Add(s_gasTransferRecipient); + if (!engine.SnapshotCache.Contains(key)) + { + engine.SnapshotCache.Add(key, new StorageItem(System.Array.Empty())); + } + } + + private static void SeedGasAccount(DataCache snapshot, UInt160 account, BigInteger balance) + { + var key = new KeyBuilder(NativeContract.GAS.Id, TokenAccountPrefix).Add(account); + var item = snapshot.GetAndChange(key, () => new StorageItem(new AccountState())); + item.GetInteroperable().Balance = balance; + } + + private static byte[] CreateBlsScalar() + { + var scalar = new byte[32]; + scalar[0] = 0x03; + return scalar; + } + + private static void EmitCryptoBlsDeserializeCall(InstructionBuilder builder, byte[] payload) + { + builder.Push(payload); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + builder.Push((int)CallFlags.All); + builder.Push("bls12381Deserialize"); + builder.Push(NativeContract.CryptoLib.Hash.GetSpan().ToArray()); + builder.AddInstruction(new Instruction + { + _opCode = VM.OpCode.SYSCALL, + _operand = BitConverter.GetBytes(ApplicationEngine.System_Contract_Call.Hash) + }); + } + + private static string ToBase64Url(string value) + { + var bytes = System.Text.Encoding.UTF8.GetBytes(value); + var base64 = Convert.ToBase64String(bytes); + return base64.TrimEnd('=').Replace('+', '-').Replace('/', '_'); + } + + private static void EmitPolicyAttributeFeeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push((byte)TransactionAttributeType.HighPriority); + } + + private static void EmitPolicyIsBlockedArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(CommitteeAddress.GetSpan().ToArray()); + } + + private static void EmitCryptoHashArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(CreateBytePayload(profile, 0x50)); + } + + private static void EmitCryptoMurmurArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(CreateBytePayload(profile, 0x51)); + builder.Push((uint)0x12345678); + } + + private static void EmitCryptoBlsDeserializeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + EmitPackedSingleArgument(builder, s_blsG1); + } + + private static void EmitCryptoBlsSerializeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + EmitCryptoBlsDeserializeCall(builder, s_blsG1); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitCryptoBlsAddArguments(InstructionBuilder builder, ScenarioProfile profile) + { + EmitCryptoBlsDeserializeCall(builder, s_blsGt); + EmitCryptoBlsDeserializeCall(builder, s_blsGt); + builder.Push(2); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitCryptoBlsEqualArguments(InstructionBuilder builder, ScenarioProfile profile) + { + EmitCryptoBlsDeserializeCall(builder, s_blsG1); + EmitCryptoBlsDeserializeCall(builder, s_blsG1); + builder.Push(2); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitCryptoBlsMulArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(false); + builder.Push(s_blsScalar); + EmitCryptoBlsDeserializeCall(builder, s_blsGt); + builder.Push(3); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitCryptoBlsPairingArguments(InstructionBuilder builder, ScenarioProfile profile) + { + EmitCryptoBlsDeserializeCall(builder, s_blsG2); + EmitCryptoBlsDeserializeCall(builder, s_blsG1); + builder.Push(2); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitCryptoRecoverSecpArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_recoverSignature); + builder.Push(s_recoverMessageHash); + builder.Push(2); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitCryptoVerifyEcdsaArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push((int)NamedCurveHash.secp256k1SHA256); + builder.Push(s_verifySignature); + builder.Push(s_verifyPubKey); + builder.Push(s_verifyMessage); + builder.Push(4); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitCryptoVerifyEd25519Arguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_ed25519Signature); + builder.Push(s_ed25519PublicKey); + builder.Push(System.Array.Empty()); + builder.Push(3); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitStdLibBase58EncodeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(CreateBytePayload(profile, 0x10)); + } + + private static void EmitStdLibBase58DecodeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(Base58.Encode(CreateBytePayload(profile, 0x11))); + } + + private static void EmitStdLibBase58CheckEncodeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(CreateBytePayload(profile, 0x12)); + } + + private static void EmitStdLibBase58CheckDecodeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(Base58.Base58CheckEncode(CreateBytePayload(profile, 0x13))); + } + + private static void EmitStdLibBase64EncodeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(CreateBytePayload(profile, 0x14)); + } + + private static void EmitStdLibBase64DecodeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(Convert.ToBase64String(CreateBytePayload(profile, 0x15))); + } + + private static void EmitStdLibBase64UrlEncodeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(CreateStringPayload(profile, seed: 'n')); + } + + private static void EmitStdLibBase64UrlDecodeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(ToBase64Url(CreateStringPayload(profile, seed: 'n'))); + } + + private static void EmitStdLibHexEncodeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(CreateBytePayload(profile, 0x16)); + } + + private static void EmitStdLibHexDecodeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(Convert.ToHexString(CreateBytePayload(profile, 0x17)).ToLowerInvariant()); + } + + private static void EmitStdLibMemoryCompareArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(CreateBytePayload(profile, 0x18)); + builder.Push(CreateBytePayload(profile, 0x28)); + } + + private static void EmitStdLibMemorySearchArguments(InstructionBuilder builder, ScenarioProfile profile) + { + var haystack = CreateBytePayload(profile, 0x30, scale: 1.5, minLength: 4); + var needleLength = Math.Clamp(haystack.Length / Math.Max(2, profile.CollectionLength == 0 ? 2 : profile.CollectionLength), 1, haystack.Length); + var needle = haystack.Take(needleLength).ToArray(); + builder.Push(haystack); + builder.Push(needle); + } + + private static void EmitStdLibStringSplitArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(CreateDelimitedString(profile)); + builder.Push("-"); + } + + private static void EmitStdLibSerializeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(CreateBytePayload(profile, 0x40)); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitStdLibDeserializeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(CreateBytePayload(profile, 0x41)); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + builder.Push((int)CallFlags.All); + builder.Push("serialize"); + builder.Push(NativeContract.StdLib.Hash.GetSpan().ToArray()); + builder.AddInstruction(new Instruction + { + _opCode = VM.OpCode.SYSCALL, + _operand = BitConverter.GetBytes(ApplicationEngine.System_Contract_Call.Hash) + }); + builder.AddInstruction(VM.OpCode.PUSH1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitStdLibJsonSerializeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(CreateStringPayload(profile, seed: 'j')); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitStdLibJsonDeserializeArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(Encoding.UTF8.GetBytes(CreateJsonPayload(profile))); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitStdLibStrLenArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(CreateStringPayload(profile)); + builder.Push(1); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitStdLibAtoiArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(CreateNumericPayload(profile)); + } + + private static void EmitStdLibItoaArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(Math.Max(1, profile.DataLength)); + } + + private static ECPoint GetCommitteePublicKey() + { + var committee = s_standbyCommittee; + if (committee.Count > 0) + return committee[0]; + var validators = s_standbyValidators; + if (validators.Count > 0) + return validators[0]; + return ECCurve.Secp256r1.G; + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/native/NativeSuite.cs b/benchmarks/Neo.VM.Benchmarks/native/NativeSuite.cs new file mode 100644 index 0000000000..f622dbcec4 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/native/NativeSuite.cs @@ -0,0 +1,67 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeSuite.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using BenchmarkDotNet.Attributes; +using Neo.VM.Benchmark.Infrastructure; +using System; +using System.Collections.Generic; +using System.IO; + +namespace Neo.VM.Benchmark.Native +{ + public class NativeSuite : VmBenchmarkSuite + { + private BenchmarkResultRecorder? _recorder; + private string? _artifactPath; + + [GlobalSetup] + public void SuiteSetup() + { + _recorder = new BenchmarkResultRecorder(); + BenchmarkExecutionContext.CurrentRecorder = _recorder; + var root = Environment.GetEnvironmentVariable("NEO_BENCHMARK_ARTIFACTS") + ?? Path.Combine(AppContext.BaseDirectory, "BenchmarkArtifacts"); + Directory.CreateDirectory(root); + _artifactPath = Path.Combine(root, $"native-metrics-{DateTime.UtcNow:yyyyMMddHHmmss}.csv"); + } + + [IterationSetup] + public void IterationSetup() + { + if (_recorder is not null) + BenchmarkExecutionContext.CurrentRecorder = _recorder; + } + + [GlobalCleanup] + public void SuiteCleanup() + { + if (_recorder is null || string.IsNullOrEmpty(_artifactPath)) + return; + + var summary = new BenchmarkExecutionSummary(_recorder, _artifactPath); + summary.Write(); + BenchmarkArtifactRegistry.RegisterMetrics(BenchmarkComponent.NativeContract, _artifactPath); + + var coverageRoot = Environment.GetEnvironmentVariable("NEO_BENCHMARK_ARTIFACTS") + ?? Path.Combine(AppContext.BaseDirectory, "BenchmarkArtifacts"); + var coveragePath = Path.Combine(coverageRoot, "native-missing.csv"); + var missingNative = NativeCoverageReport.GetMissing(); + InteropCoverageReport.WriteReport(coveragePath, System.Array.Empty(), missingNative); + BenchmarkArtifactRegistry.RegisterCoverage("native-missing", coveragePath); + + } + + protected override IEnumerable GetCases() + { + return NativeScenarioFactory.CreateCases(); + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/opcodes/OpcodeCoverageReport.cs b/benchmarks/Neo.VM.Benchmarks/opcodes/OpcodeCoverageReport.cs new file mode 100644 index 0000000000..d831ea5d6f --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/opcodes/OpcodeCoverageReport.cs @@ -0,0 +1,54 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// OpcodeCoverageReport.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using System.IO; +using System.Text; + +namespace Neo.VM.Benchmark.OpCode +{ + internal static class OpcodeCoverageReport + { + public static IReadOnlyCollection GetUncoveredOpcodes() + { + var covered = OpcodeScenarioFactory.GetSupportedOpcodes(); + return Enum.GetValues() + .Where(op => op != VM.OpCode.ABORT && !covered.Contains(op)) + .ToArray(); + } + + public static void WriteCoverageTable(string path) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var builder = new StringBuilder(); + builder.AppendLine("Opcode,Covered"); + var covered = OpcodeScenarioFactory.GetSupportedOpcodes(); + foreach (var opcode in Enum.GetValues().OrderBy(op => op)) + { + builder.Append(opcode); + builder.Append(','); + builder.Append(covered.Contains(opcode) ? "yes" : "no"); + builder.AppendLine(); + } + File.WriteAllText(path, builder.ToString()); + } + + public static void WriteMissingList(string path, IReadOnlyCollection missing) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + using var writer = new StreamWriter(path, append: false); + writer.WriteLine("Opcode"); + foreach (var opcode in missing.OrderBy(static op => op)) + { + writer.WriteLine(opcode); + } + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/opcodes/OpcodeScenarioFactory.cs b/benchmarks/Neo.VM.Benchmarks/opcodes/OpcodeScenarioFactory.cs new file mode 100644 index 0000000000..4230066058 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/opcodes/OpcodeScenarioFactory.cs @@ -0,0 +1,2041 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// OpcodeScenarioFactory.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo.VM; +using Neo.VM.Benchmark; +using Neo.VM.Benchmark.Infrastructure; +using Neo.VM.Types; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Neo.VM.Benchmark.OpCode +{ + internal static class OpcodeScenarioFactory + { + private sealed record OpcodeScenarioBuilder(Func Factory) + { + public VmBenchmarkCase BuildCase(VM.OpCode opcode, ScenarioComplexity complexity) + { + var profile = ScenarioProfile.For(complexity); + return new VmBenchmarkCase(opcode.ToString(), BenchmarkComponent.Opcode, complexity, Factory(profile)); + } + } + + public static IEnumerable CreateCases() + { + var builders = CreateBuilders(); + foreach (ScenarioComplexity complexity in Enum.GetValues()) + { + foreach (var (opcode, builder) in builders.OrderBy(static kv => kv.Key)) + { + yield return builder.BuildCase(opcode, complexity); + } + } + } + + public static IReadOnlyCollection GetSupportedOpcodes() + { + return CreateBuilders().Keys.ToArray(); + } + + private static IReadOnlyDictionary CreateBuilders() + { + var map = new Dictionary(); + RegisterPush(map); + RegisterStack(map); + RegisterArithmetic(map); + RegisterBitwise(map); + RegisterLogic(map); + RegisterSplice(map); + RegisterCompound(map); + RegisterSlots(map); + RegisterTypes(map); + RegisterControl(map); + return map; + } + + #region Registry helpers + + private static void RegisterPush(IDictionary map) + { + foreach (var opcode in Enum.GetValues()) + { + if (IsScalarPush(opcode)) + map[opcode] = new OpcodeScenarioBuilder(profile => CreateScalarPushScenario(opcode, profile)); + else if (opcode is VM.OpCode.PUSHNULL) + map[opcode] = new OpcodeScenarioBuilder(CreatePushNullScenario); + else if (opcode is VM.OpCode.PUSHINT8 or VM.OpCode.PUSHINT16 or VM.OpCode.PUSHINT32 or VM.OpCode.PUSHINT64) + map[opcode] = new OpcodeScenarioBuilder(profile => CreateNumericPushScenario(opcode, profile)); + else if (opcode is VM.OpCode.PUSHINT128 or VM.OpCode.PUSHINT256) + map[opcode] = new OpcodeScenarioBuilder(profile => CreateBigIntegerPushScenario(opcode, profile)); + else if (opcode is VM.OpCode.PUSHDATA1 or VM.OpCode.PUSHDATA2 or VM.OpCode.PUSHDATA4) + map[opcode] = new OpcodeScenarioBuilder(profile => CreatePushDataScenario(opcode, profile)); + else if (opcode is VM.OpCode.PUSHA) + map[opcode] = new OpcodeScenarioBuilder(profile => CreatePointerPushScenario(profile)); + } + } + + private static void RegisterStack(IDictionary map) + { + map[VM.OpCode.DEPTH] = new OpcodeScenarioBuilder(profile => CreateDepthScenario(profile)); + map[VM.OpCode.DROP] = new OpcodeScenarioBuilder(profile => CreateDropScenario(profile)); + map[VM.OpCode.DUP] = new OpcodeScenarioBuilder(profile => CreateDupScenario(profile)); + map[VM.OpCode.SWAP] = new OpcodeScenarioBuilder(profile => CreateSwapScenario(profile)); + map[VM.OpCode.CLEAR] = new OpcodeScenarioBuilder(profile => CreateClearScenario(profile)); + map[VM.OpCode.NIP] = new OpcodeScenarioBuilder(profile => CreateNipScenario(profile)); + map[VM.OpCode.XDROP] = new OpcodeScenarioBuilder(profile => CreateXDropScenario(profile)); + map[VM.OpCode.OVER] = new OpcodeScenarioBuilder(profile => CreateOverScenario(profile)); + map[VM.OpCode.PICK] = new OpcodeScenarioBuilder(profile => CreatePickScenario(profile)); + map[VM.OpCode.TUCK] = new OpcodeScenarioBuilder(profile => CreateTuckScenario(profile)); + map[VM.OpCode.ROT] = new OpcodeScenarioBuilder(profile => CreateRotScenario(profile)); + map[VM.OpCode.ROLL] = new OpcodeScenarioBuilder(profile => CreateRollScenario(profile)); + map[VM.OpCode.REVERSE3] = new OpcodeScenarioBuilder(profile => CreateReverse3Scenario(profile)); + map[VM.OpCode.REVERSE4] = new OpcodeScenarioBuilder(profile => CreateReverse4Scenario(profile)); + map[VM.OpCode.REVERSEN] = new OpcodeScenarioBuilder(profile => CreateReverseNScenario(profile)); + } + + private static void RegisterArithmetic(IDictionary map) + { + foreach (var opcode in new[] { VM.OpCode.SIGN, VM.OpCode.ABS, VM.OpCode.NEGATE, VM.OpCode.INC, VM.OpCode.DEC, VM.OpCode.NZ }) + { + map[opcode] = new OpcodeScenarioBuilder(profile => CreateUnaryNumericScenario(opcode, profile)); + } + + foreach (var opcode in new[] { VM.OpCode.ADD, VM.OpCode.SUB, VM.OpCode.MUL, VM.OpCode.DIV, VM.OpCode.MOD, VM.OpCode.MIN, VM.OpCode.MAX, VM.OpCode.MODMUL }) + { + map[opcode] = new OpcodeScenarioBuilder(profile => CreateBinaryNumericScenario(opcode, profile)); + } + + map[VM.OpCode.SHL] = new OpcodeScenarioBuilder(profile => CreateShiftScenario(VM.OpCode.SHL, profile)); + map[VM.OpCode.SHR] = new OpcodeScenarioBuilder(profile => CreateShiftScenario(VM.OpCode.SHR, profile)); + map[VM.OpCode.POW] = new OpcodeScenarioBuilder(profile => CreateBinaryNumericScenario(VM.OpCode.POW, profile, left: 3, right: 3)); + map[VM.OpCode.SQRT] = new OpcodeScenarioBuilder(profile => CreateUnaryNumericScenario(VM.OpCode.SQRT, profile, operand: 16)); + map[VM.OpCode.MODPOW] = new OpcodeScenarioBuilder(profile => CreateTernaryNumericScenario(VM.OpCode.MODPOW, profile)); + } + + private static void RegisterBitwise(IDictionary map) + { + map[VM.OpCode.INVERT] = new OpcodeScenarioBuilder(profile => CreateInvertScenario(profile)); + foreach (var opcode in new[] { VM.OpCode.AND, VM.OpCode.OR, VM.OpCode.XOR }) + { + map[opcode] = new OpcodeScenarioBuilder(profile => CreateBinaryNumericScenario(opcode, profile)); + } + map[VM.OpCode.NOT] = new OpcodeScenarioBuilder(profile => CreateUnaryNumericScenario(VM.OpCode.NOT, profile)); + } + + private static void RegisterLogic(IDictionary map) + { + foreach (var opcode in new[] { VM.OpCode.BOOLAND, VM.OpCode.BOOLOR }) + { + map[opcode] = new OpcodeScenarioBuilder(profile => CreateBooleanBinaryScenario(opcode, profile)); + } + + foreach (var opcode in new[] { VM.OpCode.EQUAL, VM.OpCode.NOTEQUAL, VM.OpCode.NUMEQUAL, VM.OpCode.NUMNOTEQUAL, VM.OpCode.LT, VM.OpCode.LE, VM.OpCode.GT, VM.OpCode.GE }) + { + map[opcode] = new OpcodeScenarioBuilder(profile => CreateBinaryNumericScenario(opcode, profile)); + } + + map[VM.OpCode.WITHIN] = new OpcodeScenarioBuilder(profile => CreateWithinScenario(profile)); + } + + private static void RegisterSplice(IDictionary map) + { + map[VM.OpCode.NEWBUFFER] = new OpcodeScenarioBuilder(profile => CreateNewBufferScenario(profile)); + map[VM.OpCode.MEMCPY] = new OpcodeScenarioBuilder(profile => CreateMemcpyScenario(profile)); + map[VM.OpCode.CAT] = new OpcodeScenarioBuilder(profile => CreateConcatScenario(profile)); + map[VM.OpCode.SUBSTR] = new OpcodeScenarioBuilder(profile => CreateSubstringScenario(VM.OpCode.SUBSTR, profile)); + map[VM.OpCode.LEFT] = new OpcodeScenarioBuilder(profile => CreateSubstringScenario(VM.OpCode.LEFT, profile)); + map[VM.OpCode.RIGHT] = new OpcodeScenarioBuilder(profile => CreateSubstringScenario(VM.OpCode.RIGHT, profile)); + } + + private static void RegisterCompound(IDictionary map) + { + map[VM.OpCode.PACK] = new OpcodeScenarioBuilder(profile => CreatePackScenario(VM.OpCode.PACK, profile)); + map[VM.OpCode.PACKSTRUCT] = new OpcodeScenarioBuilder(profile => CreatePackScenario(VM.OpCode.PACKSTRUCT, profile)); + map[VM.OpCode.PACKMAP] = new OpcodeScenarioBuilder(profile => CreatePackMapScenario(profile)); + map[VM.OpCode.UNPACK] = new OpcodeScenarioBuilder(profile => CreateUnpackScenario(profile)); + map[VM.OpCode.NEWARRAY0] = new OpcodeScenarioBuilder(profile => CreateSimpleAllocationScenario(VM.OpCode.NEWARRAY0, profile)); + map[VM.OpCode.NEWSTRUCT0] = new OpcodeScenarioBuilder(profile => CreateSimpleAllocationScenario(VM.OpCode.NEWSTRUCT0, profile)); + map[VM.OpCode.NEWARRAY] = new OpcodeScenarioBuilder(profile => CreateNewArrayScenario(VM.OpCode.NEWARRAY, profile)); + map[VM.OpCode.NEWARRAY_T] = new OpcodeScenarioBuilder(profile => CreateNewArrayTypedScenario(profile)); + map[VM.OpCode.NEWSTRUCT] = new OpcodeScenarioBuilder(profile => CreateNewArrayScenario(VM.OpCode.NEWSTRUCT, profile)); + map[VM.OpCode.NEWMAP] = new OpcodeScenarioBuilder(profile => CreateSimpleAllocationScenario(VM.OpCode.NEWMAP, profile)); + map[VM.OpCode.SIZE] = new OpcodeScenarioBuilder(profile => CreateSizeScenario(profile)); + map[VM.OpCode.HASKEY] = new OpcodeScenarioBuilder(profile => CreateHasKeyScenario(profile)); + map[VM.OpCode.KEYS] = new OpcodeScenarioBuilder(profile => CreateKeysScenario(profile)); + map[VM.OpCode.VALUES] = new OpcodeScenarioBuilder(profile => CreateValuesScenario(profile)); + map[VM.OpCode.PICKITEM] = new OpcodeScenarioBuilder(profile => CreatePickItemScenario(profile)); + map[VM.OpCode.SETITEM] = new OpcodeScenarioBuilder(profile => CreateSetItemScenario(profile)); + map[VM.OpCode.REMOVE] = new OpcodeScenarioBuilder(profile => CreateRemoveScenario(profile)); + map[VM.OpCode.CLEARITEMS] = new OpcodeScenarioBuilder(profile => CreateClearItemsScenario(profile)); + map[VM.OpCode.REVERSEITEMS] = new OpcodeScenarioBuilder(profile => CreateReverseItemsScenario(profile)); + map[VM.OpCode.APPEND] = new OpcodeScenarioBuilder(profile => CreateAppendScenario(profile)); + map[VM.OpCode.POPITEM] = new OpcodeScenarioBuilder(profile => CreatePopItemScenario(profile)); + } + + private static void RegisterSlots(IDictionary map) + { + map[VM.OpCode.INITSLOT] = new OpcodeScenarioBuilder(profile => CreateInitSlotScenario(VM.OpCode.INITSLOT, profile)); + map[VM.OpCode.INITSSLOT] = new OpcodeScenarioBuilder(profile => CreateInitSlotScenario(VM.OpCode.INITSSLOT, profile)); + + RegisterLoadStores(map, + new[] { VM.OpCode.LDLOC0, VM.OpCode.LDLOC1, VM.OpCode.LDLOC2, VM.OpCode.LDLOC3, VM.OpCode.LDLOC4, VM.OpCode.LDLOC5, VM.OpCode.LDLOC6 }, + VM.OpCode.LDLOC, + CreateLocalLoadScenario); + + RegisterLoadStores(map, + new[] { VM.OpCode.STLOC0, VM.OpCode.STLOC1, VM.OpCode.STLOC2, VM.OpCode.STLOC3, VM.OpCode.STLOC4, VM.OpCode.STLOC5, VM.OpCode.STLOC6 }, + VM.OpCode.STLOC, + CreateLocalStoreScenario); + + RegisterLoadStores(map, + new[] { VM.OpCode.LDARG0, VM.OpCode.LDARG1, VM.OpCode.LDARG2, VM.OpCode.LDARG3, VM.OpCode.LDARG4, VM.OpCode.LDARG5, VM.OpCode.LDARG6 }, + VM.OpCode.LDARG, + CreateArgumentLoadScenario); + + RegisterLoadStores(map, + new[] { VM.OpCode.STARG0, VM.OpCode.STARG1, VM.OpCode.STARG2, VM.OpCode.STARG3, VM.OpCode.STARG4, VM.OpCode.STARG5, VM.OpCode.STARG6 }, + VM.OpCode.STARG, + CreateArgumentStoreScenario); + + RegisterLoadStores(map, + new[] { VM.OpCode.LDSFLD0, VM.OpCode.LDSFLD1, VM.OpCode.LDSFLD2, VM.OpCode.LDSFLD3, VM.OpCode.LDSFLD4, VM.OpCode.LDSFLD5, VM.OpCode.LDSFLD6 }, + VM.OpCode.LDSFLD, + CreateStaticLoadScenario); + + RegisterLoadStores(map, + new[] { VM.OpCode.STSFLD0, VM.OpCode.STSFLD1, VM.OpCode.STSFLD2, VM.OpCode.STSFLD3, VM.OpCode.STSFLD4, VM.OpCode.STSFLD5, VM.OpCode.STSFLD6 }, + VM.OpCode.STSFLD, + CreateStaticStoreScenario); + } + + private static void RegisterTypes(IDictionary map) + { + map[VM.OpCode.ISNULL] = new OpcodeScenarioBuilder(profile => CreateIsNullScenario(profile)); + map[VM.OpCode.ISTYPE] = new OpcodeScenarioBuilder(profile => CreateIsTypeScenario(profile)); + map[VM.OpCode.CONVERT] = new OpcodeScenarioBuilder(profile => CreateConvertScenario(profile)); + } + + private static void RegisterControl(IDictionary map) + { + map[VM.OpCode.NOP] = new OpcodeScenarioBuilder(profile => CreateSimpleControlScenario(VM.OpCode.NOP, profile)); + map[VM.OpCode.RET] = new OpcodeScenarioBuilder(profile => CreateSimpleControlScenario(VM.OpCode.RET, profile)); + + foreach (var opcode in new[] { VM.OpCode.JMP, VM.OpCode.JMP_L }) + map[opcode] = new OpcodeScenarioBuilder(profile => CreateJumpScenario(opcode, profile)); + + foreach (var opcode in new[] + { + VM.OpCode.JMPIF, + VM.OpCode.JMPIF_L, + VM.OpCode.JMPIFNOT, + VM.OpCode.JMPIFNOT_L, + VM.OpCode.JMPEQ, + VM.OpCode.JMPEQ_L, + VM.OpCode.JMPNE, + VM.OpCode.JMPNE_L, + VM.OpCode.JMPGT, + VM.OpCode.JMPGT_L, + VM.OpCode.JMPGE, + VM.OpCode.JMPGE_L, + VM.OpCode.JMPLT, + VM.OpCode.JMPLT_L, + VM.OpCode.JMPLE, + VM.OpCode.JMPLE_L + }) + { + map[opcode] = new OpcodeScenarioBuilder(profile => CreateConditionalJumpScenario(opcode, profile)); + } + + foreach (var opcode in new[] { VM.OpCode.CALL, VM.OpCode.CALL_L, VM.OpCode.CALLA }) + map[opcode] = new OpcodeScenarioBuilder(profile => CreateCallScenario(opcode, profile)); + + map[VM.OpCode.CALLT] = new OpcodeScenarioBuilder(profile => CreateCallTSimpleScenario(profile)); + map[VM.OpCode.ABORT] = new OpcodeScenarioBuilder(profile => CreateAbortScenario(VM.OpCode.ABORT, profile)); + map[VM.OpCode.ABORTMSG] = new OpcodeScenarioBuilder(profile => CreateAbortScenario(VM.OpCode.ABORTMSG, profile)); + map[VM.OpCode.ASSERT] = new OpcodeScenarioBuilder(profile => CreateAssertScenario(VM.OpCode.ASSERT, profile)); + map[VM.OpCode.ASSERTMSG] = new OpcodeScenarioBuilder(profile => CreateAssertScenario(VM.OpCode.ASSERTMSG, profile)); + map[VM.OpCode.THROW] = new OpcodeScenarioBuilder(profile => CreateThrowScenario(profile)); + map[VM.OpCode.TRY] = new OpcodeScenarioBuilder(profile => CreateTryScenario(VM.OpCode.TRY, profile)); + map[VM.OpCode.TRY_L] = new OpcodeScenarioBuilder(profile => CreateTryScenario(VM.OpCode.TRY_L, profile)); + map[VM.OpCode.ENDTRY] = new OpcodeScenarioBuilder(profile => CreateSimpleControlScenario(VM.OpCode.ENDTRY, profile)); + map[VM.OpCode.ENDTRY_L] = new OpcodeScenarioBuilder(profile => CreateSimpleControlScenario(VM.OpCode.ENDTRY_L, profile)); + map[VM.OpCode.ENDFINALLY] = new OpcodeScenarioBuilder(profile => CreateSimpleControlScenario(VM.OpCode.ENDFINALLY, profile)); + map[VM.OpCode.SYSCALL] = new OpcodeScenarioBuilder(profile => CreateSyscallScenario(profile)); + } + + private static void RegisterLoadStores( + IDictionary map, + IReadOnlyList fixedOpcodes, + VM.OpCode paramOpcode, + Func factory) + { + for (byte i = 0; i < fixedOpcodes.Count; i++) + { + var index = i; + var fixedOpcode = fixedOpcodes[index]; + map[fixedOpcode] = new OpcodeScenarioBuilder(profile => factory(fixedOpcode, profile, index, null)); + } + map[paramOpcode] = new OpcodeScenarioBuilder(profile => factory(paramOpcode, profile, 2, 2)); + } + + #endregion + + #region Scenario creators + + private static OpcodeVmScenario CreateScalarPushScenario(VM.OpCode opcode, ScenarioProfile profile) + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.AddInstruction(VM.OpCode.PUSH0); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => builder.AddInstruction(opcode)); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => builder.AddInstruction(opcode)); + return CreateScenario(opcode, profile, baseline, single, saturated, + after: (engine, instruction) => { if (instruction.OpCode == opcode) engine.Pop(); }); + } + + private static OpcodeVmScenario CreateNumericPushScenario(VM.OpCode opcode, ScenarioProfile profile) + { + int size = opcode switch + { + VM.OpCode.PUSHINT8 => 1, + VM.OpCode.PUSHINT16 => 2, + VM.OpCode.PUSHINT32 => 4, + VM.OpCode.PUSHINT64 => 8, + _ => throw new ArgumentOutOfRangeException(nameof(opcode)) + }; + var operand = new byte[size]; + operand[^1] = 0x01; + return CreatePushWithOperand(opcode, profile, operand); + } + + private static OpcodeVmScenario CreateBigIntegerPushScenario(VM.OpCode opcode, ScenarioProfile profile) + { + int size = opcode == VM.OpCode.PUSHINT128 ? 16 : 32; + var operand = new byte[size]; + operand[^1] = 0x01; + return CreatePushWithOperand(opcode, profile, operand); + } + + private static OpcodeVmScenario CreatePushDataScenario(VM.OpCode opcode, ScenarioProfile profile) + { + int payloadSize = opcode switch + { + VM.OpCode.PUSHDATA1 => Math.Min(profile.DataLength, byte.MaxValue), + VM.OpCode.PUSHDATA2 => Math.Clamp(profile.DataLength * 4, byte.MaxValue + 1, ushort.MaxValue), + VM.OpCode.PUSHDATA4 => Math.Clamp(profile.DataLength * 16, ushort.MaxValue + 1, 128 * 1024), + _ => throw new ArgumentOutOfRangeException(nameof(opcode)) + }; + var payload = Enumerable.Range(0, payloadSize).Select(i => (byte)(i % 256)).ToArray(); + var operand = opcode switch + { + VM.OpCode.PUSHDATA1 => BuildOperand(1, payload), + VM.OpCode.PUSHDATA2 => BuildOperand(2, payload), + VM.OpCode.PUSHDATA4 => BuildOperand(4, payload), + _ => throw new ArgumentOutOfRangeException(nameof(opcode)) + }; + return CreatePushWithOperand(opcode, profile, operand); + } + + private static OpcodeVmScenario CreatePointerPushScenario(ScenarioProfile profile) + { + var operand = BitConverter.GetBytes(0); + return CreatePushWithOperand(VM.OpCode.PUSHA, profile, operand); + } + + private static OpcodeVmScenario CreatePushNullScenario(ScenarioProfile profile) + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.AddInstruction(VM.OpCode.PUSH0); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.AddInstruction(VM.OpCode.PUSHNULL); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.AddInstruction(VM.OpCode.PUSHNULL); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.PUSHNULL, profile, baseline, single, saturated, + after: (engine, instruction) => + { + if (instruction.OpCode == VM.OpCode.PUSHNULL) + engine.Pop(); + }); + } + + private static OpcodeVmScenario CreatePushWithOperand(VM.OpCode opcode, ScenarioProfile profile, byte[] operand) + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.AddInstruction(VM.OpCode.PUSH0); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => builder.AddInstruction(new Instruction { _opCode = opcode, _operand = operand })); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => builder.AddInstruction(new Instruction { _opCode = opcode, _operand = operand })); + return CreateScenario(opcode, profile, baseline, single, saturated, + after: (engine, instruction) => { if (instruction.OpCode == opcode) engine.Pop(); }); + } + + private static OpcodeVmScenario CreateDepthScenario(ScenarioProfile profile) + { + var baseline = CreateBaselineScript(profile, 1); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => builder.AddInstruction(VM.OpCode.DEPTH)); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => builder.AddInstruction(VM.OpCode.DEPTH)); + return CreateScenario(VM.OpCode.DEPTH, profile, baseline, single, saturated, + before: (engine, instruction) => SeedStack(engine, CreateSequentialIntegers(Math.Min(8, profile.CollectionLength))), + after: (engine, instruction) => { if (instruction.OpCode == VM.OpCode.DEPTH) engine.Pop(); }); + } + + private static OpcodeVmScenario CreateDropScenario(ScenarioProfile profile) + { + var baseline = CreateBaselineScript(profile, 1); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => builder.AddInstruction(VM.OpCode.DROP)); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => builder.AddInstruction(VM.OpCode.DROP)); + return CreateScenario(VM.OpCode.DROP, profile, baseline, single, saturated, + before: (engine, instruction) => SeedStack(engine, true)); + } + + private static OpcodeVmScenario CreateDupScenario(ScenarioProfile profile) + { + var baseline = CreateBaselineScript(profile, 1); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => builder.AddInstruction(VM.OpCode.DUP)); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => builder.AddInstruction(VM.OpCode.DUP)); + return CreateScenario(VM.OpCode.DUP, profile, baseline, single, saturated, + before: (engine, instruction) => SeedStack(engine, true), + after: (engine, instruction) => { if (instruction.OpCode == VM.OpCode.DUP) engine.Pop(); }); + } + + private static OpcodeVmScenario CreateSwapScenario(ScenarioProfile profile) + { + var baseline = CreateBaselineScript(profile, 2); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => builder.AddInstruction(VM.OpCode.SWAP)); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => builder.AddInstruction(VM.OpCode.SWAP)); + return CreateScenario(VM.OpCode.SWAP, profile, baseline, single, saturated, + before: (engine, instruction) => SeedStack(engine, true, false)); + } + + private static OpcodeVmScenario CreateClearScenario(ScenarioProfile profile) + { + var baseline = CreateBaselineScript(profile, 2); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => builder.AddInstruction(VM.OpCode.CLEAR)); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => builder.AddInstruction(VM.OpCode.CLEAR)); + return CreateScenario(VM.OpCode.CLEAR, profile, baseline, single, saturated, + before: (engine, instruction) => SeedStack(engine, true, false)); + } + + private static OpcodeVmScenario CreateNipScenario(ScenarioProfile profile) + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.AddInstruction(VM.OpCode.NIP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(1); + builder.Push(2); + builder.AddInstruction(VM.OpCode.NIP); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.NIP, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateXDropScenario(ScenarioProfile profile) + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.Push(0); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.Push(1); + builder.AddInstruction(VM.OpCode.XDROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.Push(1); + builder.AddInstruction(VM.OpCode.XDROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.XDROP, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateOverScenario(ScenarioProfile profile) + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.AddInstruction(VM.OpCode.OVER); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(1); + builder.Push(2); + builder.AddInstruction(VM.OpCode.OVER); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.OVER, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreatePickScenario(ScenarioProfile profile) + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.Push(0); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.Push(2); + builder.AddInstruction(VM.OpCode.PICK); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.Push(2); + builder.AddInstruction(VM.OpCode.PICK); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.PICK, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateTuckScenario(ScenarioProfile profile) + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.AddInstruction(VM.OpCode.TUCK); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(1); + builder.Push(2); + builder.AddInstruction(VM.OpCode.TUCK); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.TUCK, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateRotScenario(ScenarioProfile profile) + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.AddInstruction(VM.OpCode.ROT); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.AddInstruction(VM.OpCode.ROT); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.ROT, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateRollScenario(ScenarioProfile profile) + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.Push(0); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.Push(2); + builder.AddInstruction(VM.OpCode.ROLL); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.Push(2); + builder.AddInstruction(VM.OpCode.ROLL); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.ROLL, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateReverse3Scenario(ScenarioProfile profile) + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.AddInstruction(VM.OpCode.REVERSE3); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.AddInstruction(VM.OpCode.REVERSE3); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.REVERSE3, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateReverse4Scenario(ScenarioProfile profile) + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.Push(4); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.Push(4); + builder.AddInstruction(VM.OpCode.REVERSE4); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(1); + builder.Push(2); + builder.Push(3); + builder.Push(4); + builder.AddInstruction(VM.OpCode.REVERSE4); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.REVERSE4, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateReverseNScenario(ScenarioProfile profile) + { + var count = Math.Clamp(profile.CollectionLength, 2, 8); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitSequentialPushes(builder, count); + builder.Push(count); + builder.AddInstruction(VM.OpCode.DROP); + for (int i = 0; i < count; i++) builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitSequentialPushes(builder, count); + builder.Push(count); + builder.AddInstruction(VM.OpCode.REVERSEN); + for (int i = 0; i < count; i++) builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + EmitSequentialPushes(builder, count); + builder.Push(count); + builder.AddInstruction(VM.OpCode.REVERSEN); + for (int i = 0; i < count; i++) builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.REVERSEN, profile, baseline, single, saturated); + } + + private static void EmitSequentialPushes(InstructionBuilder builder, int count) + { + for (int i = 0; i < count; i++) + builder.Push(i + 1); + } + + private static OpcodeVmScenario CreateUnaryNumericScenario(VM.OpCode opcode, ScenarioProfile profile, int operand = 3) + { + var baseline = CreateBaselineScript(profile, 1); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(operand); + builder.AddInstruction(opcode); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(operand); + builder.AddInstruction(opcode); + }); + return CreateScenario(opcode, profile, baseline, single, saturated, + after: (engine, instruction) => { if (instruction.OpCode == opcode) engine.Pop(); }); + } + + private static OpcodeVmScenario CreateBinaryNumericScenario(VM.OpCode opcode, ScenarioProfile profile, int left = 5, int right = 2) + { + var baseline = CreateBaselineScript(profile, 2); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(left); + builder.Push(right); + builder.AddInstruction(opcode); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(left); + builder.Push(right); + builder.AddInstruction(opcode); + }); + return CreateScenario(opcode, profile, baseline, single, saturated, + after: (engine, instruction) => { if (instruction.OpCode == opcode) engine.Pop(); }); + } + + private static OpcodeVmScenario CreateTernaryNumericScenario(VM.OpCode opcode, ScenarioProfile profile) + { + var baseline = CreateBaselineScript(profile, 3); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(5); + builder.Push(3); + builder.Push(7); + builder.AddInstruction(opcode); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(5); + builder.Push(3); + builder.Push(7); + builder.AddInstruction(opcode); + }); + return CreateScenario(opcode, profile, baseline, single, saturated, + after: (engine, instruction) => { if (instruction.OpCode == opcode) engine.Pop(); }); + } + + private static OpcodeVmScenario CreateShiftScenario(VM.OpCode opcode, ScenarioProfile profile) + { + var baseline = CreateBaselineScript(profile, 2); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(10); + builder.Push(1); + builder.AddInstruction(opcode); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(10); + builder.Push(1); + builder.AddInstruction(opcode); + }); + return CreateScenario(opcode, profile, baseline, single, saturated, + after: (engine, instruction) => { if (instruction.OpCode == opcode) engine.Pop(); }); + } + + private static OpcodeVmScenario CreateInvertScenario(ScenarioProfile profile) + { + var data = Enumerable.Repeat((byte)0xAA, Math.Max(8, profile.DataLength / 4)).ToArray(); + var baseline = CreateBaselineScript(profile, 1); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(data); + builder.AddInstruction(VM.OpCode.INVERT); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(data); + builder.AddInstruction(VM.OpCode.INVERT); + }); + return CreateScenario(VM.OpCode.INVERT, profile, baseline, single, saturated, + after: (engine, instruction) => { if (instruction.OpCode == VM.OpCode.INVERT) engine.Pop(); }); + } + + private static OpcodeVmScenario CreateBooleanBinaryScenario(VM.OpCode opcode, ScenarioProfile profile) + { + var baseline = CreateBaselineScript(profile, 2); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(true); + builder.Push(false); + builder.AddInstruction(opcode); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(true); + builder.Push(false); + builder.AddInstruction(opcode); + }); + return CreateScenario(opcode, profile, baseline, single, saturated, + after: (engine, instruction) => { if (instruction.OpCode == opcode) engine.Pop(); }); + } + + private static OpcodeVmScenario CreateWithinScenario(ScenarioProfile profile) + { + var baseline = CreateBaselineScript(profile, 3); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(2); + builder.Push(1); + builder.Push(4); + builder.AddInstruction(VM.OpCode.WITHIN); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(2); + builder.Push(1); + builder.Push(4); + builder.AddInstruction(VM.OpCode.WITHIN); + }); + return CreateScenario(VM.OpCode.WITHIN, profile, baseline, single, saturated, + after: (engine, instruction) => { if (instruction.OpCode == VM.OpCode.WITHIN) engine.Pop(); }); + } + + private static OpcodeVmScenario CreateNewBufferScenario(ScenarioProfile profile) + { + var size = Math.Max(16, profile.DataLength); + var baseline = CreateBaselineScript(profile, 1); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(size); + builder.AddInstruction(VM.OpCode.NEWBUFFER); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(size); + builder.AddInstruction(VM.OpCode.NEWBUFFER); + }); + return CreateScenario(VM.OpCode.NEWBUFFER, profile, baseline, single, saturated, + after: (engine, instruction) => { if (instruction.OpCode == VM.OpCode.NEWBUFFER) engine.Pop(); }); + } + + private static OpcodeVmScenario CreateMemcpyScenario(ScenarioProfile profile) + { + var size = Math.Max(16, profile.DataLength / 2); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(new byte[size]); + builder.Push(0); + builder.Push(new byte[size]); + builder.Push(0); + builder.Push(size); + for (int i = 0; i < 5; i++) builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(new byte[size]); + builder.Push(0); + builder.Push(new byte[size]); + builder.Push(0); + builder.Push(size); + builder.AddInstruction(VM.OpCode.MEMCPY); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(new byte[size]); + builder.Push(0); + builder.Push(new byte[size]); + builder.Push(0); + builder.Push(size); + builder.AddInstruction(VM.OpCode.MEMCPY); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.MEMCPY, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateConcatScenario(ScenarioProfile profile) + { + var size = Math.Max(16, profile.DataLength / 4); + var payloadA = Enumerable.Repeat((byte)0xAA, size).ToArray(); + var payloadB = Enumerable.Repeat((byte)0xBB, size).ToArray(); + var baseline = CreateBaselineScript(profile, 2); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(payloadA); + builder.Push(payloadB); + builder.AddInstruction(VM.OpCode.CAT); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(payloadA); + builder.Push(payloadB); + builder.AddInstruction(VM.OpCode.CAT); + }); + return CreateScenario(VM.OpCode.CAT, profile, baseline, single, saturated, + after: (engine, instruction) => { if (instruction.OpCode == VM.OpCode.CAT) engine.Pop(); }); + } + + private static OpcodeVmScenario CreateSubstringScenario(VM.OpCode opcode, ScenarioProfile profile) + { + var dataLength = Math.Max(32, profile.DataLength); + var payload = Enumerable.Range(0, dataLength).Select(i => (byte)(i % 256)).ToArray(); + var count = Math.Max(8, dataLength / 4); + var baseline = CreateBaselineScript(profile, opcode == VM.OpCode.SUBSTR ? 3 : 2); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(payload); + if (opcode == VM.OpCode.SUBSTR) + { + builder.Push(0); + builder.Push(count); + } + else + { + builder.Push(count); + } + builder.AddInstruction(opcode); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(payload); + if (opcode == VM.OpCode.SUBSTR) + { + builder.Push(0); + builder.Push(count); + } + else + { + builder.Push(count); + } + builder.AddInstruction(opcode); + }); + return CreateScenario(opcode, profile, baseline, single, saturated, + after: (engine, instruction) => { if (instruction.OpCode == opcode) engine.Pop(); }); + } + + private static OpcodeVmScenario CreatePackScenario(VM.OpCode opcode, ScenarioProfile profile) + { + var values = GetSampleArrayValues(profile); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + foreach (var value in values) builder.Push(value); + builder.Push(values.Length); + for (var i = 0; i < values.Length + 1; i++) builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + foreach (var value in values) builder.Push(value); + builder.Push(values.Length); + builder.AddInstruction(opcode); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + foreach (var value in values) builder.Push(value); + builder.Push(values.Length); + builder.AddInstruction(opcode); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(opcode, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreatePackMapScenario(ScenarioProfile profile) + { + var entries = GetSampleMapEntries(); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + foreach (var entry in entries) + { + builder.Push(entry.value); + builder.Push(entry.key); + } + builder.Push(entries.Count); + for (int i = 0; i < entries.Count * 2 + 1; i++) builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + foreach (var entry in entries) + { + builder.Push(entry.value); + builder.Push(entry.key); + } + builder.Push(entries.Count); + builder.AddInstruction(VM.OpCode.PACKMAP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + foreach (var entry in entries) + { + builder.Push(entry.value); + builder.Push(entry.key); + } + builder.Push(entries.Count); + builder.AddInstruction(VM.OpCode.PACKMAP); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.PACKMAP, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateUnpackScenario(ScenarioProfile profile) + { + var values = GetSampleArrayValues(profile); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + foreach (var value in values) builder.Push(value); + builder.Push(values.Length); + builder.AddInstruction(VM.OpCode.PACK); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + foreach (var value in values) builder.Push(value); + builder.Push(values.Length); + builder.AddInstruction(VM.OpCode.PACK); + builder.AddInstruction(VM.OpCode.UNPACK); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + foreach (var value in values) builder.Push(value); + builder.Push(values.Length); + builder.AddInstruction(VM.OpCode.PACK); + builder.AddInstruction(VM.OpCode.UNPACK); + }); + return CreateScenario(VM.OpCode.UNPACK, profile, baseline, single, saturated, + after: (engine, instruction) => + { + if (instruction.OpCode != VM.OpCode.UNPACK) return; + var count = (int)engine.Pop().GetInteger(); + for (int i = 0; i < count; i++) engine.Pop(); + }); + } + + private static OpcodeVmScenario CreateSimpleAllocationScenario(VM.OpCode opcode, ScenarioProfile profile) + { + var baseline = CreateBaselineScript(profile, 0); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.AddInstruction(opcode); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.AddInstruction(opcode); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(opcode, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateNewArrayScenario(VM.OpCode opcode, ScenarioProfile profile) + { + var length = Math.Clamp(profile.CollectionLength, 1, 32); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(length); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(length); + builder.AddInstruction(opcode); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(length); + builder.AddInstruction(opcode); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(opcode, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateNewArrayTypedScenario(ScenarioProfile profile) + { + var length = Math.Clamp(profile.CollectionLength, 1, 32); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push((byte)StackItemType.Integer); + builder.Push(length); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push((byte)StackItemType.Integer); + builder.Push(length); + builder.AddInstruction(VM.OpCode.NEWARRAY_T); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push((byte)StackItemType.Integer); + builder.Push(length); + builder.AddInstruction(VM.OpCode.NEWARRAY_T); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.NEWARRAY_T, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateSizeScenario(ScenarioProfile profile) + { + var values = GetSampleArrayValues(profile); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedArray(builder, values); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedArray(builder, values); + builder.AddInstruction(VM.OpCode.SIZE); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + EmitPackedArray(builder, values); + builder.AddInstruction(VM.OpCode.SIZE); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.SIZE, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateHasKeyScenario(ScenarioProfile profile) + { + var entries = GetSampleMapEntries(); + var key = entries[0].key; + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedMap(builder, entries); + builder.Push(key); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedMap(builder, entries); + builder.Push(key); + builder.AddInstruction(VM.OpCode.HASKEY); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + EmitPackedMap(builder, entries); + builder.Push(key); + builder.AddInstruction(VM.OpCode.HASKEY); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.HASKEY, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateKeysScenario(ScenarioProfile profile) + { + var entries = GetSampleMapEntries(); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedMap(builder, entries); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedMap(builder, entries); + builder.AddInstruction(VM.OpCode.KEYS); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + EmitPackedMap(builder, entries); + builder.AddInstruction(VM.OpCode.KEYS); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.KEYS, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateValuesScenario(ScenarioProfile profile) + { + var entries = GetSampleMapEntries(); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedMap(builder, entries); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedMap(builder, entries); + builder.AddInstruction(VM.OpCode.VALUES); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + EmitPackedMap(builder, entries); + builder.AddInstruction(VM.OpCode.VALUES); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.VALUES, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreatePickItemScenario(ScenarioProfile profile) + { + var values = GetSampleArrayValues(profile); + var index = 1 % values.Length; + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedArray(builder, values); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedArray(builder, values); + builder.Push(index); + builder.AddInstruction(VM.OpCode.PICKITEM); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + EmitPackedArray(builder, values); + builder.Push(index); + builder.AddInstruction(VM.OpCode.PICKITEM); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.PICKITEM, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateSetItemScenario(ScenarioProfile profile) + { + var values = GetSampleArrayValues(profile); + var index = 1 % values.Length; + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedArray(builder, values); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedArray(builder, values); + builder.Push(index); + builder.Push(99); + builder.AddInstruction(VM.OpCode.SETITEM); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + EmitPackedArray(builder, values); + builder.Push(index); + builder.Push(99); + builder.AddInstruction(VM.OpCode.SETITEM); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.SETITEM, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateRemoveScenario(ScenarioProfile profile) + { + var values = GetSampleArrayValues(profile); + var index = 1 % values.Length; + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedArray(builder, values); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedArray(builder, values); + builder.Push(index); + builder.AddInstruction(VM.OpCode.REMOVE); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + EmitPackedArray(builder, values); + builder.Push(index); + builder.AddInstruction(VM.OpCode.REMOVE); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.REMOVE, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateClearItemsScenario(ScenarioProfile profile) + { + var values = GetSampleArrayValues(profile); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedArray(builder, values); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedArray(builder, values); + builder.AddInstruction(VM.OpCode.CLEARITEMS); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + EmitPackedArray(builder, values); + builder.AddInstruction(VM.OpCode.CLEARITEMS); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.CLEARITEMS, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateReverseItemsScenario(ScenarioProfile profile) + { + var values = GetSampleArrayValues(profile); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedArray(builder, values); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedArray(builder, values); + builder.AddInstruction(VM.OpCode.REVERSEITEMS); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + EmitPackedArray(builder, values); + builder.AddInstruction(VM.OpCode.REVERSEITEMS); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.REVERSEITEMS, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateAppendScenario(ScenarioProfile profile) + { + var values = GetSampleArrayValues(profile); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedArray(builder, values); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedArray(builder, values); + builder.Push(99); + builder.AddInstruction(VM.OpCode.APPEND); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + EmitPackedArray(builder, values); + builder.Push(99); + builder.AddInstruction(VM.OpCode.APPEND); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.APPEND, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreatePopItemScenario(ScenarioProfile profile) + { + var values = GetSampleArrayValues(profile); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedArray(builder, values); + builder.AddInstruction(VM.OpCode.DROP); + }); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitPackedArray(builder, values); + builder.AddInstruction(VM.OpCode.POPITEM); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + EmitPackedArray(builder, values); + builder.AddInstruction(VM.OpCode.POPITEM); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.POPITEM, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateInitSlotScenario(VM.OpCode opcode, ScenarioProfile profile) + { + var operand = opcode == VM.OpCode.INITSLOT ? new byte[] { 1, 0 } : new byte[] { 1 }; + var script = BuildScript(builder => + { + builder.AddInstruction(new Instruction { _opCode = opcode, _operand = operand }); + builder.AddInstruction(VM.OpCode.RET); + }); + var baseline = BuildRetScript(); + return CreateScenario(opcode, profile, baseline, script, script); + } + + private static OpcodeVmScenario CreateLocalLoadScenario(VM.OpCode opcode, ScenarioProfile profile, byte index, byte? operand) + { + var baseline = CreateBaselineScript(profile, 1); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitInstruction(builder, opcode, operand); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + EmitInstruction(builder, opcode, operand); + builder.AddInstruction(VM.OpCode.DROP); + }); + var seeded = false; + return CreateScenario(opcode, profile, baseline, single, saturated, + before: (engine, instruction) => + { + if (seeded) return; + PrepareLocalSlot(engine, index, 7); + seeded = true; + }); + } + + private static OpcodeVmScenario CreateLocalStoreScenario(VM.OpCode opcode, ScenarioProfile profile, byte index, byte? operand) + { + var baseline = CreateBaselineScript(profile, 1); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(8); + EmitInstruction(builder, opcode, operand); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(8); + EmitInstruction(builder, opcode, operand); + }); + var seeded = false; + return CreateScenario(opcode, profile, baseline, single, saturated, + before: (engine, instruction) => + { + if (seeded) return; + EnsureLocalSlot(engine, index); + seeded = true; + }); + } + + private static OpcodeVmScenario CreateArgumentLoadScenario(VM.OpCode opcode, ScenarioProfile profile, byte index, byte? operand) + { + var baseline = CreateBaselineScript(profile, 1); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitInstruction(builder, opcode, operand); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + EmitInstruction(builder, opcode, operand); + builder.AddInstruction(VM.OpCode.DROP); + }); + var seeded = false; + return CreateScenario(opcode, profile, baseline, single, saturated, + before: (engine, instruction) => + { + if (seeded) return; + PrepareArgumentSlot(engine, index, 3); + seeded = true; + }); + } + + private static OpcodeVmScenario CreateArgumentStoreScenario(VM.OpCode opcode, ScenarioProfile profile, byte index, byte? operand) + { + var baseline = CreateBaselineScript(profile, 1); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(4); + EmitInstruction(builder, opcode, operand); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(4); + EmitInstruction(builder, opcode, operand); + }); + var seeded = false; + return CreateScenario(opcode, profile, baseline, single, saturated, + before: (engine, instruction) => + { + if (seeded) return; + EnsureArgumentSlot(engine, index); + seeded = true; + }); + } + + private static OpcodeVmScenario CreateStaticLoadScenario(VM.OpCode opcode, ScenarioProfile profile, byte index, byte? operand) + { + var baseline = CreateBaselineScript(profile, 1); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + EmitInstruction(builder, opcode, operand); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + EmitInstruction(builder, opcode, operand); + builder.AddInstruction(VM.OpCode.DROP); + }); + var seeded = false; + return CreateScenario(opcode, profile, baseline, single, saturated, + before: (engine, instruction) => + { + if (seeded) return; + PrepareStaticSlot(engine, index, 9); + seeded = true; + }); + } + + private static OpcodeVmScenario CreateStaticStoreScenario(VM.OpCode opcode, ScenarioProfile profile, byte index, byte? operand) + { + var baseline = CreateBaselineScript(profile, 1); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(6); + EmitInstruction(builder, opcode, operand); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(6); + EmitInstruction(builder, opcode, operand); + }); + var seeded = false; + return CreateScenario(opcode, profile, baseline, single, saturated, + before: (engine, instruction) => + { + if (seeded) return; + EnsureStaticSlot(engine, index); + seeded = true; + }); + } + + private static OpcodeVmScenario CreateIsNullScenario(ScenarioProfile profile) + { + var baseline = CreateBaselineScript(profile, 1); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.AddInstruction(VM.OpCode.PUSHNULL); + builder.AddInstruction(VM.OpCode.ISNULL); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.AddInstruction(VM.OpCode.PUSHNULL); + builder.AddInstruction(VM.OpCode.ISNULL); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.ISNULL, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateIsTypeScenario(ScenarioProfile profile) + { + var baseline = CreateBaselineScript(profile, 1); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(5); + builder.AddInstruction(new Instruction + { + _opCode = VM.OpCode.ISTYPE, + _operand = new[] { (byte)StackItemType.Integer } + }); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(5); + builder.AddInstruction(new Instruction + { + _opCode = VM.OpCode.ISTYPE, + _operand = new[] { (byte)StackItemType.Integer } + }); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.ISTYPE, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateConvertScenario(ScenarioProfile profile) + { + var baseline = CreateBaselineScript(profile, 1); + var single = LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.Push(5); + builder.AddInstruction(new Instruction + { + _opCode = VM.OpCode.CONVERT, + _operand = new[] { (byte)StackItemType.ByteString } + }); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturated = LoopScriptFactory.BuildInfiniteLoop(builder => + { + builder.Push(5); + builder.AddInstruction(new Instruction + { + _opCode = VM.OpCode.CONVERT, + _operand = new[] { (byte)StackItemType.ByteString } + }); + builder.AddInstruction(VM.OpCode.DROP); + }); + return CreateScenario(VM.OpCode.CONVERT, profile, baseline, single, saturated); + } + + private static OpcodeVmScenario CreateSimpleControlScenario(VM.OpCode opcode, ScenarioProfile profile) + { + var baseline = BuildRetScript(); + var script = BuildScript(builder => + { + builder.AddInstruction(opcode); + builder.AddInstruction(VM.OpCode.RET); + }); + return CreateScenario(opcode, profile, baseline, script, script); + } + + private static OpcodeVmScenario CreateJumpScenario(VM.OpCode opcode, ScenarioProfile profile) + { + var builder = new InstructionBuilder(); + var offset = opcode == VM.OpCode.JMP ? new[] { unchecked((byte)0x02) } : BitConverter.GetBytes(2); + builder.AddInstruction(new Instruction { _opCode = opcode, _operand = offset }); + builder.AddInstruction(VM.OpCode.NOP); + builder.AddInstruction(VM.OpCode.RET); + var script = builder.ToArray(); + var baseline = BuildRetScript(); + return CreateScenario(opcode, profile, baseline, script, script); + } + + private static OpcodeVmScenario CreateConditionalJumpScenario(VM.OpCode opcode, ScenarioProfile profile) + { + var offset = opcode switch + { + VM.OpCode.JMPIF or VM.OpCode.JMPIFNOT or VM.OpCode.JMPEQ or VM.OpCode.JMPNE or VM.OpCode.JMPGT or VM.OpCode.JMPGE or VM.OpCode.JMPLT or VM.OpCode.JMPLE => new[] { unchecked((byte)0x02) }, + _ => BitConverter.GetBytes(2) + }; + var operands = GetConditionalOperands(opcode); + var script = BuildScript(builder => + { + foreach (var value in operands) + builder.Push(value); + builder.AddInstruction(new Instruction { _opCode = opcode, _operand = offset }); + builder.AddInstruction(VM.OpCode.NOP); + builder.AddInstruction(VM.OpCode.RET); + }); + var baseline = BuildRetScript(); + return CreateScenario(opcode, profile, baseline, script, script); + } + + private static OpcodeVmScenario CreateCallScenario(VM.OpCode opcode, ScenarioProfile profile) + { + var builder = new InstructionBuilder(); + var method = builder.AddInstruction(VM.OpCode.NOP); + builder.AddInstruction(VM.OpCode.RET); + var methodOffset = method._offset; + + builder.AddInstruction(VM.OpCode.PUSH0); + switch (opcode) + { + case VM.OpCode.CALL: + builder.AddInstruction(new Instruction + { + _opCode = VM.OpCode.CALL, + _operand = new[] { unchecked((byte)(methodOffset - builder._instructions[^1]._offset - 2)) } + }); + break; + case VM.OpCode.CALL_L: + builder.AddInstruction(new Instruction + { + _opCode = VM.OpCode.CALL_L, + _operand = BitConverter.GetBytes(methodOffset - builder._instructions[^1]._offset - 5) + }); + break; + case VM.OpCode.CALLA: + builder.AddInstruction(new Instruction + { + _opCode = VM.OpCode.PUSHA, + _operand = BitConverter.GetBytes(methodOffset) + }); + builder.AddInstruction(VM.OpCode.CALLA); + break; + case VM.OpCode.CALLT: + builder.AddInstruction(new Instruction + { + _opCode = VM.OpCode.CALLT, + _operand = BitConverter.GetBytes((ushort)0) + }); + break; + } + builder.AddInstruction(VM.OpCode.RET); + var script = builder.ToArray(); + var baseline = BuildRetScript(); + return CreateScenario(opcode, profile, baseline, script, script); + } + + private static OpcodeVmScenario CreateCallTSimpleScenario(ScenarioProfile profile) + { + var script = BuildScript(builder => + { + builder.AddInstruction(VM.OpCode.CALLT); + builder.AddInstruction(VM.OpCode.RET); + }); + var baseline = BuildRetScript(); + return CreateScenario(VM.OpCode.CALLT, profile, baseline, script, script); + } + + private static OpcodeVmScenario CreateAbortScenario(VM.OpCode opcode, ScenarioProfile profile) + { + var script = BuildScript(builder => + { + builder.AddInstruction(opcode); + builder.AddInstruction(VM.OpCode.RET); + }); + var baseline = BuildRetScript(); + return CreateScenario(opcode, profile, baseline, script, script); + } + + private static OpcodeVmScenario CreateAssertScenario(VM.OpCode opcode, ScenarioProfile profile) + { + var script = BuildScript(builder => + { + builder.AddInstruction(VM.OpCode.PUSHT); + builder.AddInstruction(opcode); + builder.AddInstruction(VM.OpCode.RET); + }); + var baseline = BuildRetScript(); + return CreateScenario(opcode, profile, baseline, script, script); + } + + private static OpcodeVmScenario CreateThrowScenario(ScenarioProfile profile) + { + var script = BuildScript(builder => + { + builder.AddInstruction(VM.OpCode.THROW); + builder.AddInstruction(VM.OpCode.RET); + }); + var baseline = BuildRetScript(); + return CreateScenario(VM.OpCode.THROW, profile, baseline, script, script); + } + + private static OpcodeVmScenario CreateTryScenario(VM.OpCode opcode, ScenarioProfile profile) + { + var script = BuildScript(builder => + { + builder.AddInstruction(new Instruction + { + _opCode = opcode, + _operand = opcode == VM.OpCode.TRY ? new byte[] { 0, 0 } : new byte[8] + }); + builder.AddInstruction(VM.OpCode.RET); + }); + var baseline = BuildRetScript(); + return CreateScenario(opcode, profile, baseline, script, script); + } + + private static OpcodeVmScenario CreateSyscallScenario(ScenarioProfile profile) + { + var script = BuildScript(builder => + { + builder.AddInstruction(new Instruction + { + _opCode = VM.OpCode.SYSCALL, + _operand = BitConverter.GetBytes(0x77777777) + }); + builder.AddInstruction(VM.OpCode.RET); + }); + var baseline = BuildRetScript(); + return CreateScenario(VM.OpCode.SYSCALL, profile, baseline, script, script); + } + + #endregion + + #region Helper methods + + private static OpcodeVmScenario CreateScenario( + VM.OpCode opcode, + ScenarioProfile profile, + byte[] baselineScript, + byte[] singleScript, + byte[] saturatedScript, + Action? before = null, + Action? after = null) + { + long price = Benchmark_Opcode.OpCodePrices.TryGetValue(opcode, out var p) ? p : 1; + long budget = Math.Max(price * profile.Iterations * 8L, Benchmark_Opcode.OneGasDatoshi); + return new OpcodeVmScenario(baselineScript, singleScript, saturatedScript, budget, before, after); + } + + private static bool IsScalarPush(VM.OpCode opcode) => + opcode is VM.OpCode.PUSHT or VM.OpCode.PUSHF or VM.OpCode.PUSHM1 || + (opcode >= VM.OpCode.PUSH0 && opcode <= VM.OpCode.PUSH16); + + private static byte[] BuildOperand(int prefixSize, byte[] payload) + { + var operand = new byte[prefixSize + payload.Length]; + switch (prefixSize) + { + case 1: + operand[0] = (byte)payload.Length; + break; + case 2: + BitConverter.GetBytes((ushort)payload.Length).CopyTo(operand, 0); + break; + case 4: + BitConverter.GetBytes((uint)payload.Length).CopyTo(operand, 0); + break; + } + System.Buffer.BlockCopy(payload, 0, operand, prefixSize, payload.Length); + return operand; + } + + private static byte[] CreateBaselineScript(ScenarioProfile profile, int pushCount) + { + return LoopScriptFactory.BuildCountingLoop(profile, builder => + { + for (int i = 0; i < pushCount; i++) + builder.Push(0); + for (int i = 0; i < pushCount; i++) + builder.AddInstruction(VM.OpCode.DROP); + }); + } + + private static void SeedStack(BenchmarkEngine engine, params object[] items) + { + var stack = engine.CurrentContext!.EvaluationStack; + stack.Clear(); + foreach (var item in items) + engine.Push(StackItem.FromInterface(item)); + } + + private static object[] CreateSequentialIntegers(int count) => + Enumerable.Range(0, count).Select(i => (object)i).ToArray(); + + private static void PrepareLocalSlot(BenchmarkEngine engine, int index, object value) + { + var slot = EnsureLocalSlot(engine, index); + slot[index] = StackItem.FromInterface(value); + } + + private static Slot EnsureLocalSlot(BenchmarkEngine engine, int index) + { + var context = engine.CurrentContext!; + var required = index + 1; + var slot = EnsureSlot(context.LocalVariables, required, engine.ReferenceCounter); + context.LocalVariables = slot; + return slot; + } + + private static Slot EnsureArgumentSlot(BenchmarkEngine engine, int index) + { + var context = engine.CurrentContext!; + var required = index + 1; + var slot = EnsureSlot(context.Arguments, required, engine.ReferenceCounter); + context.Arguments = slot; + return slot; + } + + private static void PrepareArgumentSlot(BenchmarkEngine engine, int index, object value) + { + var slot = EnsureArgumentSlot(engine, index); + slot[index] = StackItem.FromInterface(value); + } + + private static Slot EnsureStaticSlot(BenchmarkEngine engine, int index) + { + var context = engine.CurrentContext!; + var required = index + 1; + var slot = EnsureSlot(context.StaticFields, required, engine.ReferenceCounter); + context.StaticFields = slot; + return slot; + } + + private static void PrepareStaticSlot(BenchmarkEngine engine, int index, object value) + { + var slot = EnsureStaticSlot(engine, index); + slot[index] = StackItem.FromInterface(value); + } + + private static Slot EnsureSlot(Slot? existing, int requiredSize, IReferenceCounter referenceCounter) + { + if (existing is null) + return new Slot(requiredSize, referenceCounter); + if (existing.Count >= requiredSize) + return existing; + existing.ClearReferences(); + return new Slot(requiredSize, referenceCounter); + } + + private static byte[] BuildRetScript() => BuildScript(builder => builder.AddInstruction(VM.OpCode.RET)); + + private static byte[] BuildScript(Action emitter) + { + var builder = new InstructionBuilder(); + emitter(builder); + return builder.ToArray(); + } + + private static void EmitInstruction(InstructionBuilder builder, VM.OpCode opcode, byte? operand) + { + if (operand is null) + { + builder.AddInstruction(opcode); + } + else + { + builder.AddInstruction(new Instruction + { + _opCode = opcode, + _operand = new[] { operand.Value } + }); + } + } + + private static object[] GetConditionalOperands(VM.OpCode opcode) => opcode switch + { + VM.OpCode.JMPIF or VM.OpCode.JMPIF_L => new object[] { true }, + VM.OpCode.JMPIFNOT or VM.OpCode.JMPIFNOT_L => new object[] { false }, + VM.OpCode.JMPEQ or VM.OpCode.JMPEQ_L => new object[] { 3, 3 }, + VM.OpCode.JMPNE or VM.OpCode.JMPNE_L => new object[] { 3, 4 }, + VM.OpCode.JMPGT or VM.OpCode.JMPGT_L => new object[] { 5, 3 }, + VM.OpCode.JMPGE or VM.OpCode.JMPGE_L => new object[] { 3, 3 }, + VM.OpCode.JMPLT or VM.OpCode.JMPLT_L => new object[] { 2, 4 }, + VM.OpCode.JMPLE or VM.OpCode.JMPLE_L => new object[] { 3, 3 }, + _ => System.Array.Empty() + }; + + private static object[] GetSampleArrayValues(ScenarioProfile profile) + { + var count = Math.Clamp(profile.CollectionLength, 2, 4); + return Enumerable.Range(1, count).Select(i => (object)i).ToArray(); + } + + private static IReadOnlyList<(object key, object value)> GetSampleMapEntries() + { + return new List<(object key, object value)> + { + ("a", 1), + ("b", 2) + }; + } + + private static void EmitPackedArray(InstructionBuilder builder, IReadOnlyList values) + { + foreach (var value in values) builder.Push(value); + builder.Push(values.Count); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitPackedMap(InstructionBuilder builder, IReadOnlyList<(object key, object value)> entries) + { + foreach (var (key, value) in entries) + { + builder.Push(value); + builder.Push(key); + } + builder.Push(entries.Count); + builder.AddInstruction(VM.OpCode.PACKMAP); + } + + #endregion + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/opcodes/OpcodeSuite.cs b/benchmarks/Neo.VM.Benchmarks/opcodes/OpcodeSuite.cs new file mode 100644 index 0000000000..9ae0221fb5 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/opcodes/OpcodeSuite.cs @@ -0,0 +1,67 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// OpcodeSuite.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using BenchmarkDotNet.Attributes; +using Neo.VM.Benchmark.Infrastructure; +using System; +using System.IO; + +namespace Neo.VM.Benchmark.OpCode +{ + public class OpcodeSuite : VmBenchmarkSuite + { + private BenchmarkResultRecorder? _recorder; + private string? _artifactPath; + private string? _coveragePath; + + [GlobalSetup] + public void SuiteSetup() + { + _recorder = new BenchmarkResultRecorder(); + BenchmarkExecutionContext.CurrentRecorder = _recorder; + var root = Environment.GetEnvironmentVariable("NEO_BENCHMARK_ARTIFACTS") + ?? Path.Combine(AppContext.BaseDirectory, "BenchmarkArtifacts"); + Directory.CreateDirectory(root); + _artifactPath = Path.Combine(root, $"opcode-metrics-{DateTime.UtcNow:yyyyMMddHHmmss}.csv"); + _coveragePath = Path.Combine(root, "opcode-coverage.csv"); + } + + [IterationSetup] + public void IterationSetup() + { + if (_recorder is not null) + BenchmarkExecutionContext.CurrentRecorder = _recorder; + } + + [GlobalCleanup] + public void SuiteCleanup() + { + if (_recorder is null || string.IsNullOrEmpty(_artifactPath)) + return; + var summary = new BenchmarkExecutionSummary(_recorder, _artifactPath); + summary.Write(); + BenchmarkArtifactRegistry.RegisterMetrics(BenchmarkComponent.Opcode, _artifactPath); + + if (!string.IsNullOrEmpty(_coveragePath)) + { + var missing = OpcodeCoverageReport.GetUncoveredOpcodes(); + OpcodeCoverageReport.WriteCoverageTable(_coveragePath); + if (missing.Count > 0) + { + Console.WriteLine($"[OpcodeSuite] Missing coverage for {missing.Count} opcodes: {string.Join(", ", missing)}"); + } + BenchmarkArtifactRegistry.RegisterCoverage("opcode-coverage", _coveragePath); + } + } + + protected override IEnumerable GetCases() => OpcodeScenarioFactory.CreateCases(); + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/syscalls/SyscallCoverageReport.cs b/benchmarks/Neo.VM.Benchmarks/syscalls/SyscallCoverageReport.cs new file mode 100644 index 0000000000..838e3e31f1 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/syscalls/SyscallCoverageReport.cs @@ -0,0 +1,25 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// SyscallCoverageReport.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo.VM.Benchmark.Infrastructure; +using System.Collections.Generic; + +namespace Neo.VM.Benchmark.Syscalls +{ + internal static class SyscallCoverageReport + { + public static IReadOnlyCollection GetMissing() + { + var covered = SyscallCoverageTracker.GetCovered(); + return InteropCoverageReport.MissingSyscalls(covered); + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/syscalls/SyscallCoverageTracker.cs b/benchmarks/Neo.VM.Benchmarks/syscalls/SyscallCoverageTracker.cs new file mode 100644 index 0000000000..3931562348 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/syscalls/SyscallCoverageTracker.cs @@ -0,0 +1,37 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// SyscallCoverageTracker.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using System.Collections.Generic; +using System.Linq; + +namespace Neo.VM.Benchmark.Syscalls +{ + internal static class SyscallCoverageTracker + { + private static readonly HashSet s_covered = new(); + + public static void Register(string name) + { + lock (s_covered) + { + s_covered.Add(name); + } + } + + public static IReadOnlyCollection GetCovered() + { + lock (s_covered) + { + return s_covered.ToArray(); + } + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/syscalls/SyscallScenarioFactory.cs b/benchmarks/Neo.VM.Benchmarks/syscalls/SyscallScenarioFactory.cs new file mode 100644 index 0000000000..a97bc59ea8 --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/syscalls/SyscallScenarioFactory.cs @@ -0,0 +1,1176 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// SyscallScenarioFactory.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo; +using Neo.Cryptography.ECC; +using Neo.Extensions; +using Neo.Json; +using Neo.Network.P2P.Payloads; +using Neo.Persistence; +using Neo.SmartContract; +using Neo.SmartContract.Manifest; +using Neo.SmartContract.Native; +using Neo.VM; +using Neo.VM.Benchmark; +using Neo.VM.Benchmark.Infrastructure; +using Neo.VM.Types; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Neo.VM.Benchmark.Syscalls +{ + internal static class SyscallScenarioFactory + { + private const byte ContractManagementPrefixContract = 8; + private const byte ContractManagementPrefixContractHash = 12; + private const int CallInvokerContractId = 0x4343; + private const int CallCalleeContractId = 0x4344; + + private static readonly byte[] s_calleeScript = BuildCalleeScript(); + private static readonly ContractState s_calleeContract = CreateScriptContract(s_calleeScript, "Callee", CallCalleeContractId, "run"); + private static readonly UInt160 s_calleeScriptHash = s_calleeScript.ToScriptHash(); + + private static readonly IReadOnlyList s_standbyValidators = BenchmarkProtocolSettings.StandbyValidators; + private static readonly byte[] s_checkSigPublicKey = s_standbyValidators[0].EncodePoint(true); + private static readonly byte[] s_checkSigSignature = new byte[64]; + + private static readonly byte[][] s_multisigSignatures = { new byte[64] }; + private static readonly byte[][] s_multisigPubKeys = s_standbyValidators + .Take(2) + .Select(validators => validators.EncodePoint(true)) + .ToArray(); + + private static readonly ContractState s_nativePolicyContract = NativeContract.Policy.GetContractState(ProtocolSettings.Default, 0); + private const int StorageContractId = 0x4242; + + private static readonly byte[] s_storageKey = { 0xB0, 0x01 }; + private static readonly byte[] s_iteratorPrefix = { 0xC1 }; + + private sealed record ScenarioBuilder( + Func ScriptFactory, + Action? Configure = null, + Func? EngineFactory = null) + { + public VmBenchmarkCase BuildCase(InteropDescriptor descriptor, ScenarioComplexity complexity) + { + var profile = ScenarioProfile.For(complexity); + var scenario = new ApplicationEngineVmScenario(ScriptFactory, Configure, EngineFactory); + return new VmBenchmarkCase(descriptor.Name, BenchmarkComponent.Syscall, complexity, scenario); + } + } + + private static readonly UInt160[] s_witnessAccounts = + { + UInt160.Parse("0x0102030405060708090a0b0c0d0e0f1011121314"), + UInt160.Parse("0x1112131415161718191a1b1c1d1e1f2021222324") + }; + + public static IEnumerable CreateCases() + { + var builders = CreateBuilders(); + foreach (ScenarioComplexity complexity in Enum.GetValues()) + { + foreach (var (descriptor, builder) in builders) + { + SyscallCoverageTracker.Register(descriptor.Name); + yield return builder.BuildCase(descriptor, complexity); + } + } + } + + private static IReadOnlyDictionary CreateBuilders() + { + var map = new Dictionary + { + [ApplicationEngine.System_Runtime_Platform] = new ScenarioBuilder(CreateZeroArgumentScripts(ApplicationEngine.System_Runtime_Platform)), + [ApplicationEngine.System_Runtime_GetNetwork] = new ScenarioBuilder(CreateZeroArgumentScripts(ApplicationEngine.System_Runtime_GetNetwork)), + [ApplicationEngine.System_Runtime_GetAddressVersion] = new ScenarioBuilder(CreateZeroArgumentScripts(ApplicationEngine.System_Runtime_GetAddressVersion)), + [ApplicationEngine.System_Runtime_GetTrigger] = new ScenarioBuilder(CreateZeroArgumentScripts(ApplicationEngine.System_Runtime_GetTrigger)), + [ApplicationEngine.System_Runtime_GetTime] = new ScenarioBuilder(CreateZeroArgumentScripts(ApplicationEngine.System_Runtime_GetTime)), + [ApplicationEngine.System_Runtime_GetInvocationCounter] = new ScenarioBuilder(CreateZeroArgumentScripts(ApplicationEngine.System_Runtime_GetInvocationCounter)), + [ApplicationEngine.System_Runtime_GasLeft] = new ScenarioBuilder(CreateZeroArgumentScripts(ApplicationEngine.System_Runtime_GasLeft)), + [ApplicationEngine.System_Runtime_GetRandom] = new ScenarioBuilder(CreateZeroArgumentScripts(ApplicationEngine.System_Runtime_GetRandom)), + [ApplicationEngine.System_Runtime_GetScriptContainer] = new ScenarioBuilder(CreateZeroArgumentScripts(ApplicationEngine.System_Runtime_GetScriptContainer), EngineFactory: CreateTransactionBackedEngine), + [ApplicationEngine.System_Runtime_GetExecutingScriptHash] = new ScenarioBuilder(CreateZeroArgumentScripts(ApplicationEngine.System_Runtime_GetExecutingScriptHash)), + [ApplicationEngine.System_Runtime_GetCallingScriptHash] = new ScenarioBuilder(CreateZeroArgumentScripts(ApplicationEngine.System_Runtime_GetCallingScriptHash)), + [ApplicationEngine.System_Runtime_GetEntryScriptHash] = new ScenarioBuilder(CreateZeroArgumentScripts(ApplicationEngine.System_Runtime_GetEntryScriptHash)), + [ApplicationEngine.System_Runtime_Log] = new ScenarioBuilder(CreateSyscallScripts(ApplicationEngine.System_Runtime_Log, EmitLogArguments, dropResult: false)), + [ApplicationEngine.System_Runtime_Notify] = new ScenarioBuilder(CreateSyscallScripts(ApplicationEngine.System_Runtime_Notify, EmitNotifyArguments, dropResult: false)), + [ApplicationEngine.System_Runtime_GetNotifications] = new ScenarioBuilder(CreateSyscallScripts(ApplicationEngine.System_Runtime_GetNotifications, EmitGetNotificationsArguments, dropResult: true)), + [ApplicationEngine.System_Runtime_BurnGas] = new ScenarioBuilder(CreateSyscallScripts(ApplicationEngine.System_Runtime_BurnGas, EmitBurnGasArguments, dropResult: false)), + [ApplicationEngine.System_Runtime_CurrentSigners] = new ScenarioBuilder(CreateZeroArgumentScripts(ApplicationEngine.System_Runtime_CurrentSigners), EngineFactory: CreateTransactionBackedEngine), + [ApplicationEngine.System_Runtime_CheckWitness] = new ScenarioBuilder(CreateSyscallScripts(ApplicationEngine.System_Runtime_CheckWitness, EmitCheckWitnessArguments, dropResult: true), EngineFactory: CreateTransactionBackedEngine), + [ApplicationEngine.System_Runtime_LoadScript] = new ScenarioBuilder(CreateLoadScriptScripts()), + [ApplicationEngine.System_Contract_GetCallFlags] = new ScenarioBuilder(CreateZeroArgumentScripts(ApplicationEngine.System_Contract_GetCallFlags)), + [ApplicationEngine.System_Contract_CreateStandardAccount] = new ScenarioBuilder(CreateSyscallScripts(ApplicationEngine.System_Contract_CreateStandardAccount, EmitCreateStandardAccountArguments, dropResult: true)), + [ApplicationEngine.System_Contract_CreateMultisigAccount] = new ScenarioBuilder(CreateSyscallScripts(ApplicationEngine.System_Contract_CreateMultisigAccount, EmitCreateMultisigAccountArguments, dropResult: true)), + [ApplicationEngine.System_Contract_Call] = new ScenarioBuilder(CreateContractCallScripts()), + [ApplicationEngine.System_Contract_CallNative] = new ScenarioBuilder(CreateContractCallNativeScripts()), + [ApplicationEngine.System_Contract_NativeOnPersist] = new ScenarioBuilder(CreateNativeOnPersistScripts(), ConfigureNativePersist, _ => BenchmarkApplicationEngine.Create(trigger: TriggerType.OnPersist)), + [ApplicationEngine.System_Contract_NativePostPersist] = new ScenarioBuilder(CreateNativePostPersistScripts(), ConfigureNativePersist, _ => BenchmarkApplicationEngine.Create(trigger: TriggerType.PostPersist)), + [ApplicationEngine.System_Crypto_CheckSig] = new ScenarioBuilder(CreateCheckSigScripts(), EngineFactory: CreateTransactionBackedEngine), + [ApplicationEngine.System_Crypto_CheckMultisig] = new ScenarioBuilder(CreateCheckMultisigScripts(), EngineFactory: CreateTransactionBackedEngine), + [ApplicationEngine.System_Storage_GetContext] = new ScenarioBuilder(CreateStorageGetContextScripts()), + [ApplicationEngine.System_Storage_GetReadOnlyContext] = new ScenarioBuilder(CreateStorageGetReadOnlyContextScripts()), + [ApplicationEngine.System_Storage_AsReadOnly] = new ScenarioBuilder(CreateStorageAsReadOnlyScripts()), + [ApplicationEngine.System_Storage_Get] = new ScenarioBuilder(CreateStorageGetScripts()), + [ApplicationEngine.System_Storage_Put] = new ScenarioBuilder(CreateStoragePutScripts()), + [ApplicationEngine.System_Storage_Delete] = new ScenarioBuilder(CreateStorageDeleteScripts()), + [ApplicationEngine.System_Storage_Find] = new ScenarioBuilder(CreateStorageFindScripts()), + [ApplicationEngine.System_Storage_Local_Get] = new ScenarioBuilder(CreateStorageLocalGetScripts()), + [ApplicationEngine.System_Storage_Local_Put] = new ScenarioBuilder(CreateStorageLocalPutScripts()), + [ApplicationEngine.System_Storage_Local_Delete] = new ScenarioBuilder(CreateStorageLocalDeleteScripts()), + [ApplicationEngine.System_Storage_Local_Find] = new ScenarioBuilder(CreateStorageLocalFindScripts()), + [ApplicationEngine.System_Iterator_Next] = new ScenarioBuilder(CreateIteratorNextScripts()), + [ApplicationEngine.System_Iterator_Value] = new ScenarioBuilder(CreateIteratorValueScripts()) + }; + + return map; + } + + private static Func CreateZeroArgumentScripts(InteropDescriptor descriptor) + { + return CreateSyscallScripts(descriptor, emitArguments: null, dropResult: true); + } + + private static Func CreateSyscallScripts( + InteropDescriptor descriptor, + Action? emitArguments, + bool dropResult, + Func? saturatedProfileSelector = null) + { + return profile => + { + var baseline = CreateScript(BuildNoOpLoop(profile), profile); + var single = CreateScript(BuildSyscallLoop(descriptor, profile, emitArguments, dropResult), profile); + var saturatedProfile = saturatedProfileSelector?.Invoke(profile) + ?? new ScenarioProfile(profile.Iterations * 8, profile.DataLength, profile.CollectionLength); + var saturated = CreateScript(BuildSyscallLoop(descriptor, saturatedProfile, emitArguments, dropResult), saturatedProfile); + return new ApplicationEngineVmScenario.ApplicationEngineScriptSet(baseline, single, saturated); + }; + } + + private static Func CreateLoadScriptScripts() + { + return profile => + { + var single = CreateScript(BuildLoadScriptLoop(profile), profile); + var saturatedProfile = new ScenarioProfile(profile.Iterations * 4, Math.Max(profile.DataLength, 1) * 2, profile.CollectionLength); + var saturated = CreateScript(BuildLoadScriptLoop(saturatedProfile), saturatedProfile); + var baseline = CreateScript(BuildNoOpLoop(profile), profile); + return new ApplicationEngineVmScenario.ApplicationEngineScriptSet(baseline, single, saturated); + }; + } + + private static Func CreateStorageGetContextScripts() + { + return profile => + { + var baseline = BuildNoOpLoop(profile); + var single = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetContext); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturatedProfile = new ScenarioProfile(profile.Iterations * 8, profile.DataLength, profile.CollectionLength); + var saturated = LoopScriptFactory.BuildCountingLoop(saturatedProfile, + iteration: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetContext); + builder.AddInstruction(VM.OpCode.DROP); + }); + + return CreateStorageScriptSet(baseline, single, saturated, profile, saturatedProfile, seed: null); + }; + } + + private static Func CreateStorageGetReadOnlyContextScripts() + { + return profile => + { + var baseline = BuildNoOpLoop(profile); + var single = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetReadOnlyContext); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturatedProfile = new ScenarioProfile(profile.Iterations * 8, profile.DataLength, profile.CollectionLength); + var saturated = LoopScriptFactory.BuildCountingLoop(saturatedProfile, + iteration: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetReadOnlyContext); + builder.AddInstruction(VM.OpCode.DROP); + }); + + return CreateStorageScriptSet(baseline, single, saturated, profile, saturatedProfile, seed: null); + }; + } + + private static Func CreateStorageAsReadOnlyScripts() + { + return profile => + { + var valueLength = Math.Max(1, profile.DataLength); + var storageValue = BenchmarkDataFactory.CreateByteArray(valueLength, 0x70); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.AddInstruction(VM.OpCode.PUSH0); + builder.AddInstruction(VM.OpCode.DROP); + }, + localCount: 2); + + var single = LoopScriptFactory.BuildCountingLoop(profile, + prolog: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetContext); + builder.AddInstruction(VM.OpCode.STLOC1); + }, + iteration: builder => + { + builder.AddInstruction(VM.OpCode.LDLOC1); + EmitSyscall(builder, ApplicationEngine.System_Storage_AsReadOnly); + builder.AddInstruction(VM.OpCode.DROP); + }, + localCount: 2); + + var saturatedProfile = new ScenarioProfile(profile.Iterations * 8, profile.DataLength, profile.CollectionLength); + var saturated = LoopScriptFactory.BuildCountingLoop(saturatedProfile, + prolog: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetContext); + builder.AddInstruction(VM.OpCode.STLOC1); + }, + iteration: builder => + { + builder.AddInstruction(VM.OpCode.LDLOC1); + EmitSyscall(builder, ApplicationEngine.System_Storage_AsReadOnly); + builder.AddInstruction(VM.OpCode.DROP); + }, + localCount: 2); + + return CreateStorageScriptSet(baseline, single, saturated, profile, saturatedProfile, seed: null); + }; + } + + private static Func CreateStorageGetScripts() + { + return profile => + { + var valueLength = Math.Max(1, profile.DataLength); + var storageValue = BenchmarkDataFactory.CreateByteArray(valueLength, 0x70); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.AddInstruction(VM.OpCode.PUSH0); + builder.AddInstruction(VM.OpCode.DROP); + }, + localCount: 2); + + var single = LoopScriptFactory.BuildCountingLoop(profile, + prolog: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetContext); + builder.AddInstruction(VM.OpCode.STLOC1); + }, + iteration: builder => + { + builder.Push(s_storageKey); + builder.AddInstruction(VM.OpCode.LDLOC1); + EmitSyscall(builder, ApplicationEngine.System_Storage_Get); + builder.AddInstruction(VM.OpCode.DROP); + }, + localCount: 2); + + var saturatedProfile = new ScenarioProfile(profile.Iterations * 8, profile.DataLength, profile.CollectionLength); + var saturated = LoopScriptFactory.BuildCountingLoop(saturatedProfile, + prolog: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetContext); + builder.AddInstruction(VM.OpCode.STLOC1); + }, + iteration: builder => + { + builder.Push(s_storageKey); + builder.AddInstruction(VM.OpCode.LDLOC1); + EmitSyscall(builder, ApplicationEngine.System_Storage_Get); + builder.AddInstruction(VM.OpCode.DROP); + }, + localCount: 2); + + return CreateStorageScriptSet(baseline, single, saturated, profile, saturatedProfile, SeedSimpleStorage); + }; + } + + private static Func CreateStoragePutScripts() + { + return profile => + { + var valueLength = Math.Max(1, profile.DataLength); + var storageValue = BenchmarkDataFactory.CreateByteArray(valueLength, 0x70); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.AddInstruction(VM.OpCode.PUSH0); + builder.AddInstruction(VM.OpCode.DROP); + }, + localCount: 2); + + var single = LoopScriptFactory.BuildCountingLoop(profile, + prolog: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetContext); + builder.AddInstruction(VM.OpCode.STLOC1); + }, + iteration: builder => + { + builder.Push(storageValue); + builder.Push(s_storageKey); + builder.AddInstruction(VM.OpCode.LDLOC1); + EmitSyscall(builder, ApplicationEngine.System_Storage_Put); + }, + localCount: 2); + + var saturatedProfile = new ScenarioProfile(profile.Iterations * 8, profile.DataLength * 2, profile.CollectionLength); + var saturatedValue = BenchmarkDataFactory.CreateByteArray(Math.Max(1, saturatedProfile.DataLength), 0x72); + var saturated = LoopScriptFactory.BuildCountingLoop(saturatedProfile, + prolog: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetContext); + builder.AddInstruction(VM.OpCode.STLOC1); + }, + iteration: builder => + { + builder.Push(saturatedValue); + builder.Push(s_storageKey); + builder.AddInstruction(VM.OpCode.LDLOC1); + EmitSyscall(builder, ApplicationEngine.System_Storage_Put); + }, + localCount: 2); + + return CreateStorageScriptSet(baseline, single, saturated, profile, saturatedProfile, SeedSimpleStorage); + }; + } + + private static Func CreateStorageDeleteScripts() + { + return profile => + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.AddInstruction(VM.OpCode.PUSH0); + builder.AddInstruction(VM.OpCode.DROP); + }, + localCount: 2); + + var single = LoopScriptFactory.BuildCountingLoop(profile, + prolog: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetContext); + builder.AddInstruction(VM.OpCode.STLOC1); + }, + iteration: builder => + { + builder.Push(s_storageKey); + builder.AddInstruction(VM.OpCode.LDLOC1); + EmitSyscall(builder, ApplicationEngine.System_Storage_Delete); + }, + localCount: 2); + + var saturatedProfile = new ScenarioProfile(profile.Iterations * 8, profile.DataLength, profile.CollectionLength); + var saturated = LoopScriptFactory.BuildCountingLoop(saturatedProfile, + prolog: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetContext); + builder.AddInstruction(VM.OpCode.STLOC1); + }, + iteration: builder => + { + builder.Push(s_storageKey); + builder.AddInstruction(VM.OpCode.LDLOC1); + EmitSyscall(builder, ApplicationEngine.System_Storage_Delete); + }, + localCount: 2); + + return CreateStorageScriptSet(baseline, single, saturated, profile, saturatedProfile, SeedSimpleStorage); + }; + } + + private static Func CreateStorageLocalGetScripts() + { + return profile => + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.AddInstruction(VM.OpCode.PUSH0); + builder.AddInstruction(VM.OpCode.DROP); + }); + + var single = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.Push(s_storageKey); + EmitSyscall(builder, ApplicationEngine.System_Storage_Local_Get); + builder.AddInstruction(VM.OpCode.DROP); + }); + + var saturatedProfile = new ScenarioProfile(profile.Iterations * 8, profile.DataLength, profile.CollectionLength); + var saturated = LoopScriptFactory.BuildCountingLoop(saturatedProfile, + iteration: builder => + { + builder.Push(s_storageKey); + EmitSyscall(builder, ApplicationEngine.System_Storage_Local_Get); + builder.AddInstruction(VM.OpCode.DROP); + }); + + return CreateStorageScriptSet(baseline, single, saturated, profile, saturatedProfile, SeedSimpleStorage); + }; + } + + private static Func CreateStorageLocalPutScripts() + { + return profile => + { + var valueLength = Math.Max(1, profile.DataLength); + var storageValue = BenchmarkDataFactory.CreateByteArray(valueLength, 0x60); + var baseline = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.AddInstruction(VM.OpCode.PUSH0); + builder.AddInstruction(VM.OpCode.DROP); + }); + + var single = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.Push(storageValue); + builder.Push(s_storageKey); + EmitSyscall(builder, ApplicationEngine.System_Storage_Local_Put); + }); + + var saturatedProfile = new ScenarioProfile(profile.Iterations * 8, profile.DataLength * 2, profile.CollectionLength); + var saturatedValue = BenchmarkDataFactory.CreateByteArray(Math.Max(1, saturatedProfile.DataLength), 0x62); + var saturated = LoopScriptFactory.BuildCountingLoop(saturatedProfile, + iteration: builder => + { + builder.Push(saturatedValue); + builder.Push(s_storageKey); + EmitSyscall(builder, ApplicationEngine.System_Storage_Local_Put); + }); + + return CreateStorageScriptSet(baseline, single, saturated, profile, saturatedProfile, SeedSimpleStorage); + }; + } + + private static Func CreateStorageLocalDeleteScripts() + { + return profile => + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.AddInstruction(VM.OpCode.PUSH0); + builder.AddInstruction(VM.OpCode.DROP); + }); + + var single = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.Push(s_storageKey); + EmitSyscall(builder, ApplicationEngine.System_Storage_Local_Delete); + }); + + var saturatedProfile = new ScenarioProfile(profile.Iterations * 8, profile.DataLength, profile.CollectionLength); + var saturated = LoopScriptFactory.BuildCountingLoop(saturatedProfile, + iteration: builder => + { + builder.Push(s_storageKey); + EmitSyscall(builder, ApplicationEngine.System_Storage_Local_Delete); + }); + + return CreateStorageScriptSet(baseline, single, saturated, profile, saturatedProfile, SeedSimpleStorage); + }; + } + + private static Func CreateStorageLocalFindScripts() + { + return profile => + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.AddInstruction(VM.OpCode.PUSH0); + builder.AddInstruction(VM.OpCode.DROP); + }); + + var single = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.Push((int)FindOptions.None); + builder.Push(s_iteratorPrefix); + EmitSyscall(builder, ApplicationEngine.System_Storage_Local_Find); + builder.AddInstruction(VM.OpCode.DROP); + }); + + var saturatedProfile = new ScenarioProfile(profile.Iterations * 8, profile.DataLength, profile.CollectionLength); + var saturated = LoopScriptFactory.BuildCountingLoop(saturatedProfile, + iteration: builder => + { + builder.Push((int)FindOptions.None); + builder.Push(s_iteratorPrefix); + EmitSyscall(builder, ApplicationEngine.System_Storage_Local_Find); + builder.AddInstruction(VM.OpCode.DROP); + }); + + return CreateStorageScriptSet(baseline, single, saturated, profile, saturatedProfile, SeedIteratorEntries); + }; + } + + private static Func CreateStorageFindScripts() + { + return profile => + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.AddInstruction(VM.OpCode.PUSH0); + builder.AddInstruction(VM.OpCode.DROP); + }, + localCount: 2); + + var single = LoopScriptFactory.BuildCountingLoop(profile, + prolog: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetContext); + builder.AddInstruction(VM.OpCode.STLOC1); + }, + iteration: builder => + { + builder.Push((int)FindOptions.None); + builder.Push(s_iteratorPrefix); + builder.AddInstruction(VM.OpCode.LDLOC1); + EmitSyscall(builder, ApplicationEngine.System_Storage_Find); + builder.AddInstruction(VM.OpCode.DROP); + }, + localCount: 2); + + var saturatedProfile = new ScenarioProfile(profile.Iterations * 8, profile.DataLength, profile.CollectionLength); + var saturated = LoopScriptFactory.BuildCountingLoop(saturatedProfile, + prolog: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetContext); + builder.AddInstruction(VM.OpCode.STLOC1); + }, + iteration: builder => + { + builder.Push((int)FindOptions.None); + builder.Push(s_iteratorPrefix); + builder.AddInstruction(VM.OpCode.LDLOC1); + EmitSyscall(builder, ApplicationEngine.System_Storage_Find); + builder.AddInstruction(VM.OpCode.DROP); + }, + localCount: 2); + + return CreateStorageScriptSet(baseline, single, saturated, profile, saturatedProfile, SeedIteratorEntries); + }; + } + + private static Func CreateIteratorNextScripts() + { + return profile => + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.AddInstruction(VM.OpCode.PUSH0); + builder.AddInstruction(VM.OpCode.DROP); + }, + localCount: 2); + + var single = LoopScriptFactory.BuildCountingLoop(profile, + prolog: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetContext); + builder.AddInstruction(VM.OpCode.STLOC1); + }, + iteration: builder => + { + builder.Push((int)FindOptions.None); + builder.Push(s_iteratorPrefix); + builder.AddInstruction(VM.OpCode.LDLOC1); + EmitSyscall(builder, ApplicationEngine.System_Storage_Find); + builder.AddInstruction(VM.OpCode.DUP); + EmitSyscall(builder, ApplicationEngine.System_Iterator_Next); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }, + localCount: 2); + + var saturatedProfile = new ScenarioProfile(profile.Iterations * 8, profile.DataLength, profile.CollectionLength); + var saturated = LoopScriptFactory.BuildCountingLoop(saturatedProfile, + prolog: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetContext); + builder.AddInstruction(VM.OpCode.STLOC1); + }, + iteration: builder => + { + builder.Push((int)FindOptions.None); + builder.Push(s_iteratorPrefix); + builder.AddInstruction(VM.OpCode.LDLOC1); + EmitSyscall(builder, ApplicationEngine.System_Storage_Find); + builder.AddInstruction(VM.OpCode.DUP); + EmitSyscall(builder, ApplicationEngine.System_Iterator_Next); + builder.AddInstruction(VM.OpCode.DROP); + builder.AddInstruction(VM.OpCode.DROP); + }, + localCount: 2); + + return CreateStorageScriptSet(baseline, single, saturated, profile, saturatedProfile, SeedIteratorEntries); + }; + } + + private static Func CreateIteratorValueScripts() + { + return profile => + { + var baseline = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.AddInstruction(VM.OpCode.PUSH0); + builder.AddInstruction(VM.OpCode.DROP); + }, + localCount: 2); + + var single = LoopScriptFactory.BuildCountingLoop(profile, + prolog: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetContext); + builder.AddInstruction(VM.OpCode.STLOC1); + }, + iteration: builder => + { + builder.Push((int)FindOptions.None); + builder.Push(s_iteratorPrefix); + builder.AddInstruction(VM.OpCode.LDLOC1); + EmitSyscall(builder, ApplicationEngine.System_Storage_Find); + builder.AddInstruction(VM.OpCode.DUP); + EmitSyscall(builder, ApplicationEngine.System_Iterator_Next); + builder.AddInstruction(VM.OpCode.DROP); + EmitSyscall(builder, ApplicationEngine.System_Iterator_Value); + builder.AddInstruction(VM.OpCode.DROP); + }, + localCount: 2); + + var saturatedProfile = new ScenarioProfile(profile.Iterations * 8, profile.DataLength, profile.CollectionLength); + var saturated = LoopScriptFactory.BuildCountingLoop(saturatedProfile, + prolog: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Storage_GetContext); + builder.AddInstruction(VM.OpCode.STLOC1); + }, + iteration: builder => + { + builder.Push((int)FindOptions.None); + builder.Push(s_iteratorPrefix); + builder.AddInstruction(VM.OpCode.LDLOC1); + EmitSyscall(builder, ApplicationEngine.System_Storage_Find); + builder.AddInstruction(VM.OpCode.DUP); + EmitSyscall(builder, ApplicationEngine.System_Iterator_Next); + builder.AddInstruction(VM.OpCode.DROP); + EmitSyscall(builder, ApplicationEngine.System_Iterator_Value); + builder.AddInstruction(VM.OpCode.DROP); + }, + localCount: 2); + + return CreateStorageScriptSet(baseline, single, saturated, profile, saturatedProfile, SeedIteratorEntries); + }; + } + + private static ApplicationEngineVmScenario.ApplicationEngineScript CreateScript(byte[] script, ScenarioProfile profile) + => new(script, profile); + + private static ApplicationEngineVmScenario.ApplicationEngineScriptSet CreateStorageScriptSet( + byte[] baseline, + byte[] single, + byte[] saturated, + ScenarioProfile profile, + ScenarioProfile saturatedProfile, + Action? seed) + { + return new ApplicationEngineVmScenario.ApplicationEngineScriptSet( + CreateStorageScript(baseline, profile, seed), + CreateStorageScript(single, profile, seed), + CreateStorageScript(saturated, saturatedProfile, seed)); + } + + private static ApplicationEngineVmScenario.ApplicationEngineScript CreateStorageScript( + byte[] script, + ScenarioProfile profile, + Action? seed) + { + var scriptHash = script.ToScriptHash(); + return new ApplicationEngineVmScenario.ApplicationEngineScript(script, profile, state => + { + var cache = state.SnapshotCache ?? throw new InvalidOperationException("Snapshot cache not initialized."); + var contract = CreateStorageContract(script); + cache.Add(StorageKey.Create(NativeContract.ContractManagement.Id, ContractManagementPrefixContract, scriptHash), StorageItem.CreateSealed(contract)); + seed?.Invoke(cache, contract, profile); + state.ScriptHash = contract.Hash; + state.Contract = contract; + state.CallFlags = CallFlags.All; + }); + } + + private static ContractState CreateStorageContract(byte[] script) + { + return CreateScriptContract(script, "BenchmarkStorage", StorageContractId, "main"); + } + + private static void SeedSimpleStorage(DataCache cache, ContractState contract, ScenarioProfile profile) + { + var valueLength = Math.Max(1, profile.DataLength); + var value = BenchmarkDataFactory.CreateByteArray(valueLength, fill: 0x42); + cache.Add(new StorageKey { Id = contract.Id, Key = s_storageKey }, new StorageItem(value)); + } + + private static void SeedIteratorEntries(DataCache cache, ContractState contract, ScenarioProfile profile) + { + var entryCount = Math.Max(1, profile.CollectionLength); + var iteratorEntries = BenchmarkDataFactory.CreateByteSegments(entryCount, Math.Max(1, profile.DataLength / entryCount)); + var prefix = s_iteratorPrefix[0]; + foreach (var (key, value) in iteratorEntries.Select((value, index) => (Key: BenchmarkDataFactory.CreateIteratorKey(index, prefix), Value: value))) + cache.Add(new StorageKey { Id = contract.Id, Key = key }, new StorageItem(value)); + } + + private static void EmitPushArray(InstructionBuilder builder, IReadOnlyList items) + { + foreach (var item in items) + builder.Push(item); + builder.Push(items.Count); + builder.AddInstruction(VM.OpCode.PACK); + } + + private static void EmitSyscall(InstructionBuilder builder, InteropDescriptor descriptor) + { + builder.AddInstruction(new Instruction + { + _opCode = VM.OpCode.SYSCALL, + _operand = BitConverter.GetBytes(descriptor.Hash) + }); + } + + private static byte[] BuildCalleeScript() + { + var builder = new InstructionBuilder(); + builder.AddInstruction(VM.OpCode.RET); + return builder.ToArray(); + } + + private static ContractState CreateScriptContract(byte[] script, string name, int id, string methodName, bool safe = true) + { + var nef = new NefFile + { + Compiler = "benchmark", + Source = string.Empty, + Tokens = System.Array.Empty(), + Script = script + }; + nef.CheckSum = NefFile.ComputeChecksum(nef); + + var method = new ContractMethodDescriptor + { + Name = methodName, + Parameters = System.Array.Empty(), + ReturnType = ContractParameterType.Void, + Offset = 0, + Safe = safe + }; + + var abi = new ContractAbi + { + Methods = new[] { method }, + Events = System.Array.Empty() + }; + + var manifest = new ContractManifest + { + Name = name, + Groups = System.Array.Empty(), + SupportedStandards = System.Array.Empty(), + Abi = abi, + Permissions = new[] { ContractPermission.DefaultPermission }, + Trusts = WildcardContainer.CreateWildcard(), + Extra = new JObject() + }; + + return new ContractState + { + Id = id, + UpdateCounter = 0, + Hash = script.ToScriptHash(), + Nef = nef, + Manifest = manifest + }; + } + + private static Func CreateContractCallScripts() + { + return profile => + { + var baseline = BuildNoOpLoop(profile); + var single = BuildContractCallLoop(profile); + var saturatedProfile = new ScenarioProfile(profile.Iterations * 4, profile.DataLength, profile.CollectionLength); + var saturated = BuildContractCallLoop(saturatedProfile); + + var baselineProfile = profile.With(dataLength: 0, collectionLength: 0); + return new ApplicationEngineVmScenario.ApplicationEngineScriptSet( + CreateContractInvokerScript(baseline, baselineProfile, includeCallee: true), + CreateContractInvokerScript(single, profile, includeCallee: true), + CreateContractInvokerScript(saturated, saturatedProfile, includeCallee: true)); + }; + } + + private static Func CreateContractCallNativeScripts() + { + return profile => + { + var baseline = BuildNoOpLoop(profile); + var single = BuildContractCallNativeLoop(profile); + var saturatedProfile = new ScenarioProfile(profile.Iterations * 4, profile.DataLength, profile.CollectionLength); + var saturated = BuildContractCallNativeLoop(saturatedProfile); + var baselineProfile = profile.With(dataLength: 0, collectionLength: 0); + return new ApplicationEngineVmScenario.ApplicationEngineScriptSet( + CreateCallNativeInvokerScript(baseline, baselineProfile), + CreateCallNativeInvokerScript(single, profile), + CreateCallNativeInvokerScript(saturated, saturatedProfile)); + }; + } + + private static Func CreateNativeOnPersistScripts() + { + return profile => + { + var baseline = BuildNoOpLoop(profile); + var single = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Contract_NativeOnPersist); + }); + var saturatedProfile = new ScenarioProfile(profile.Iterations * 2, profile.DataLength, profile.CollectionLength); + var saturated = LoopScriptFactory.BuildCountingLoop(saturatedProfile, + iteration: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Contract_NativeOnPersist); + }); + + return new ApplicationEngineVmScenario.ApplicationEngineScriptSet(baseline, single, saturated); + }; + } + + private static Func CreateNativePostPersistScripts() + { + return profile => + { + var baseline = BuildNoOpLoop(profile); + var single = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Contract_NativePostPersist); + }); + var saturatedProfile = new ScenarioProfile(profile.Iterations * 2, profile.DataLength, profile.CollectionLength); + var saturated = LoopScriptFactory.BuildCountingLoop(saturatedProfile, + iteration: builder => + { + EmitSyscall(builder, ApplicationEngine.System_Contract_NativePostPersist); + }); + + return new ApplicationEngineVmScenario.ApplicationEngineScriptSet(baseline, single, saturated); + }; + } + + private static Func CreateCheckSigScripts() + { + return profile => + { + var baseline = BuildNoOpLoop(profile); + var single = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.Push(s_checkSigSignature); + builder.Push(s_checkSigPublicKey); + EmitSyscall(builder, ApplicationEngine.System_Crypto_CheckSig); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturatedProfile = new ScenarioProfile(profile.Iterations * 4, profile.DataLength, profile.CollectionLength); + var saturated = LoopScriptFactory.BuildCountingLoop(saturatedProfile, + iteration: builder => + { + builder.Push(s_checkSigSignature); + builder.Push(s_checkSigPublicKey); + EmitSyscall(builder, ApplicationEngine.System_Crypto_CheckSig); + builder.AddInstruction(VM.OpCode.DROP); + }); + + return new ApplicationEngineVmScenario.ApplicationEngineScriptSet(baseline, single, saturated); + }; + } + + private static Func CreateCheckMultisigScripts() + { + return profile => + { + var baseline = BuildNoOpLoop(profile); + var single = LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + EmitPushArray(builder, s_multisigSignatures); + EmitPushArray(builder, s_multisigPubKeys); + EmitSyscall(builder, ApplicationEngine.System_Crypto_CheckMultisig); + builder.AddInstruction(VM.OpCode.DROP); + }); + var saturatedProfile = new ScenarioProfile(profile.Iterations * 4, profile.DataLength, profile.CollectionLength); + var saturated = LoopScriptFactory.BuildCountingLoop(saturatedProfile, + iteration: builder => + { + EmitPushArray(builder, s_multisigSignatures); + EmitPushArray(builder, s_multisigPubKeys); + EmitSyscall(builder, ApplicationEngine.System_Crypto_CheckMultisig); + builder.AddInstruction(VM.OpCode.DROP); + }); + + return new ApplicationEngineVmScenario.ApplicationEngineScriptSet(baseline, single, saturated); + }; + } + + private static void ConfigureNativePersist(BenchmarkApplicationEngine engine, ScenarioProfile profile) + { + var cache = engine.SnapshotCache; + foreach (var native in NativeContract.Contracts) + { + var state = native.GetContractState(engine.ProtocolSettings, engine.PersistingBlock?.Index ?? 0); + if (state is null) + continue; + SeedContract(cache, state); + } + } + + private static byte[] BuildContractCallLoop(ScenarioProfile profile) + { + return LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.AddInstruction(VM.OpCode.NEWARRAY0); + builder.Push((int)CallFlags.All); + builder.Push("run"); + builder.Push(s_calleeScriptHash.ToArray()); + EmitSyscall(builder, ApplicationEngine.System_Contract_Call); + builder.AddInstruction(VM.OpCode.DROP); + }); + } + + private static byte[] BuildContractCallNativeLoop(ScenarioProfile profile) + { + return LoopScriptFactory.BuildCountingLoop(profile, + iteration: builder => + { + builder.Push((byte)0); + EmitSyscall(builder, ApplicationEngine.System_Contract_CallNative); + }); + } + + private static ApplicationEngineVmScenario.ApplicationEngineScript CreateContractInvokerScript(byte[] script, ScenarioProfile profile, bool includeCallee) + { + var invokerContract = CreateScriptContract(script, "Invoker", CallInvokerContractId, "run"); + return new ApplicationEngineVmScenario.ApplicationEngineScript(script, profile, state => + { + var cache = state.SnapshotCache ?? throw new InvalidOperationException("Snapshot cache not initialized."); + SeedContract(cache, invokerContract); + if (includeCallee) + { + SeedContract(cache, s_calleeContract); + } + state.ScriptHash = invokerContract.Hash; + state.Contract = invokerContract; + state.CallFlags = CallFlags.All; + }); + } + + private static ApplicationEngineVmScenario.ApplicationEngineScript CreateCallNativeInvokerScript(byte[] script, ScenarioProfile profile) + { + return new ApplicationEngineVmScenario.ApplicationEngineScript(script, profile, state => + { + state.ScriptHash = NativeContract.Policy.Hash; + state.Contract = s_nativePolicyContract; + state.CallFlags = CallFlags.All; + }); + } + + private static void SeedContract(DataCache cache, ContractState contract) + { + var contractKey = StorageKey.Create(NativeContract.ContractManagement.Id, ContractManagementPrefixContract, contract.Hash); + if (!cache.TryGet(contractKey, out _)) + cache.Add(contractKey, StorageItem.CreateSealed(contract)); + + var hashKey = StorageKey.Create(NativeContract.ContractManagement.Id, ContractManagementPrefixContractHash, contract.Id); + if (!cache.TryGet(hashKey, out _)) + cache.Add(hashKey, new StorageItem(contract.Hash.ToArray())); + } + + private static byte[] BuildLoadScriptLoop(ScenarioProfile profile) + { + var targetBuilder = new InstructionBuilder(); + targetBuilder.AddInstruction(VM.OpCode.PUSH1); + targetBuilder.AddInstruction(VM.OpCode.RET); + var targetScript = targetBuilder.ToArray(); + + return LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.AddInstruction(VM.OpCode.NEWARRAY0); + builder.Push((int)CallFlags.All); + builder.Push(targetScript); + builder.AddInstruction(new Instruction + { + _opCode = VM.OpCode.SYSCALL, + _operand = BitConverter.GetBytes(ApplicationEngine.System_Runtime_LoadScript.Hash) + }); + builder.AddInstruction(VM.OpCode.DROP); + }); + } + + private static byte[] BuildNoOpLoop(ScenarioProfile profile) + { + return LoopScriptFactory.BuildCountingLoop(profile, builder => + { + builder.AddInstruction(VM.OpCode.NOP); + }); + } + + private static byte[] BuildSyscallLoop( + InteropDescriptor descriptor, + ScenarioProfile profile, + Action? emitArguments, + bool dropResult) + { + return LoopScriptFactory.BuildCountingLoop(profile, builder => + { + emitArguments?.Invoke(builder, profile); + builder.AddInstruction(new Instruction + { + _opCode = VM.OpCode.SYSCALL, + _operand = BitConverter.GetBytes(descriptor.Hash) + }); + + if (dropResult) + { + builder.AddInstruction(VM.OpCode.DROP); + } + }); + } + + private static void EmitLogArguments(InstructionBuilder builder, ScenarioProfile profile) + { + var messageLength = Math.Max(1, profile.DataLength); + builder.Push(BenchmarkDataFactory.CreateString(messageLength, 'l')); + } + + private static void EmitNotifyArguments(InstructionBuilder builder, ScenarioProfile profile) + { + var elementCount = Math.Clamp(profile.CollectionLength, 1, 32); + var elementSize = Math.Max(1, profile.DataLength / elementCount); + for (int i = 0; i < elementCount; i++) + { + var seed = (byte)(0x60 + i); + builder.Push(BenchmarkDataFactory.CreateByteArray(elementSize + i, seed)); + } + builder.Push(elementCount); + builder.AddInstruction(VM.OpCode.PACK); + var eventNameLength = Math.Clamp(profile.DataLength / 4 + 4, 4, 48); + builder.Push(BenchmarkDataFactory.CreateString(eventNameLength, 'e')); + } + + private static void EmitGetNotificationsArguments(InstructionBuilder builder, ScenarioProfile profile) + { + builder.Push(s_calleeScriptHash.ToArray()); + } + + private static void EmitBurnGasArguments(InstructionBuilder builder, ScenarioProfile profile) + { + var datoshi = Math.Clamp(profile.DataLength + 1, 1, 10_000); + builder.Push(datoshi); + } + + private static void EmitCheckWitnessArguments(InstructionBuilder builder, ScenarioProfile profile) + { + var account = s_witnessAccounts[profile.Iterations % s_witnessAccounts.Length]; + builder.Push(account.GetSpan().ToArray()); + } + + private static void EmitCreateStandardAccountArguments(InstructionBuilder builder, ScenarioProfile profile) + { + var validators = s_standbyValidators; + var validator = validators[profile.Iterations % validators.Count]; + builder.Push(validator.EncodePoint(true)); + } + + private static void EmitCreateMultisigAccountArguments(InstructionBuilder builder, ScenarioProfile profile) + { + var validators = s_standbyValidators; + var available = validators.Count; + var count = Math.Clamp(profile.CollectionLength, 2, Math.Min(available, 5)); + for (int i = 0; i < count; i++) + { + builder.Push(validators[i].EncodePoint(true)); + } + builder.Push(count); + builder.AddInstruction(VM.OpCode.PACK); + var threshold = Math.Clamp(count / 2, 1, count); + builder.Push(threshold); + } + + private static BenchmarkApplicationEngine CreateTransactionBackedEngine(ScenarioProfile profile) + { + var transaction = new Transaction + { + Version = 0, + Nonce = (uint)Math.Max(1, profile.Iterations), + Signers = CreateSigners(), + Witnesses = System.Array.Empty(), + Attributes = System.Array.Empty(), + Script = System.Array.Empty(), + NetworkFee = 0, + SystemFee = 0, + ValidUntilBlock = 100 + }; + + return BenchmarkApplicationEngine.Create(container: transaction); + } + + private static Signer[] CreateSigners() + { + var signers = new Signer[s_witnessAccounts.Length]; + for (int i = 0; i < s_witnessAccounts.Length; i++) + { + signers[i] = new Signer + { + Account = s_witnessAccounts[i], + Scopes = WitnessScope.CalledByEntry + }; + } + return signers; + } + } +} diff --git a/benchmarks/Neo.VM.Benchmarks/syscalls/SyscallSuite.cs b/benchmarks/Neo.VM.Benchmarks/syscalls/SyscallSuite.cs new file mode 100644 index 0000000000..3d27b298aa --- /dev/null +++ b/benchmarks/Neo.VM.Benchmarks/syscalls/SyscallSuite.cs @@ -0,0 +1,67 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// SyscallSuite.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using BenchmarkDotNet.Attributes; +using Neo.VM.Benchmark.Infrastructure; +using System; +using System.Collections.Generic; +using System.IO; + +namespace Neo.VM.Benchmark.Syscalls +{ + public class SyscallSuite : VmBenchmarkSuite + { + private BenchmarkResultRecorder? _recorder; + private string? _artifactPath; + + [GlobalSetup] + public void SuiteSetup() + { + _recorder = new BenchmarkResultRecorder(); + BenchmarkExecutionContext.CurrentRecorder = _recorder; + var root = Environment.GetEnvironmentVariable("NEO_BENCHMARK_ARTIFACTS") + ?? Path.Combine(AppContext.BaseDirectory, "BenchmarkArtifacts"); + Directory.CreateDirectory(root); + _artifactPath = Path.Combine(root, $"syscall-metrics-{DateTime.UtcNow:yyyyMMddHHmmss}.csv"); + } + + [IterationSetup] + public void IterationSetup() + { + if (_recorder is not null) + BenchmarkExecutionContext.CurrentRecorder = _recorder; + } + + [GlobalCleanup] + public void SuiteCleanup() + { + if (_recorder is null || string.IsNullOrEmpty(_artifactPath)) + return; + + var summary = new BenchmarkExecutionSummary(_recorder, _artifactPath); + summary.Write(); + BenchmarkArtifactRegistry.RegisterMetrics(BenchmarkComponent.Syscall, _artifactPath); + + var coverageRoot = Environment.GetEnvironmentVariable("NEO_BENCHMARK_ARTIFACTS") + ?? Path.Combine(AppContext.BaseDirectory, "BenchmarkArtifacts"); + var coveragePath = Path.Combine(coverageRoot, "syscall-missing.csv"); + var missing = SyscallCoverageReport.GetMissing(); + InteropCoverageReport.WriteReport(coveragePath, missing, System.Array.Empty()); + BenchmarkArtifactRegistry.RegisterCoverage("syscall-missing", coveragePath); + + } + + protected override IEnumerable GetCases() + { + return SyscallScenarioFactory.CreateCases(); + } + } +} diff --git a/docs/benchmarking/coverage.md b/docs/benchmarking/coverage.md new file mode 100644 index 0000000000..ceba7768c7 --- /dev/null +++ b/docs/benchmarking/coverage.md @@ -0,0 +1,34 @@ +# Benchmark Coverage Guidance + +The benchmark harness can report which syscalls and native contract methods are still missing automated scenarios. + +## Generating reports + +1. Build the benchmark project and run the suites with coverage reporting enabled: + + ```bash + export NEO_VM_BENCHMARK=1 + export NEO_BENCHMARK_COVERAGE=1 + dotnet run --project benchmarks/Neo.VM.Benchmarks/Neo.VM.Benchmarks.csproj + ``` + +2. After the run completes, CSV files are written to the directory specified by + `NEO_BENCHMARK_ARTIFACTS` (defaults to `BenchmarkArtifacts/` under the build output): + + - `opcode-missing.csv` – opcodes without registered scenarios. The suite also writes `opcode-coverage.csv` with the full opcode registry. + - `syscall-missing.csv` – syscalls exercised vs. the runtime registry. + - `native-missing.csv` – native contract methods touched by the suite. + - `interop-missing.csv` – combined summary written by the benchmark launcher. + + Each CSV entry contains the category (`syscall` or `native`) and the identifier that still + requires a scenario. If a file is empty, that category currently has full coverage. + +3. The console output also summarizes the missing items whenever `NEO_BENCHMARK_COVERAGE=1` is set. The command exits with a non-zero code when coverage gaps remain for opcodes, syscalls, or native methods, allowing CI pipelines to fail fast if new interops or instructions lack scenarios. + +## Keeping suites up to date + +- When adding new syscalls or native contract methods upstream, the coverage report will surface them + as missing. Add scenarios to the appropriate factory (`SyscallScenarioFactory` or + `NativeScenarioFactory`), then rerun the coverage build to confirm they disappear from the report. +- If a category deliberately omits certain items (e.g., interops that require complex state), document + the rationale beside those cases in the factories so future contributors understand what remains. diff --git a/docs/benchmarking/dynamic-pricing.md b/docs/benchmarking/dynamic-pricing.md new file mode 100644 index 0000000000..328ddf1224 --- /dev/null +++ b/docs/benchmarking/dynamic-pricing.md @@ -0,0 +1,46 @@ +# Benchmark Scaling Artifacts + +The benchmark suite now exports both raw timing metrics and scaling ratios so you can derive dynamic gas prices directly from the generated CSVs. This document summarizes what is produced and how to interpret it. + +## Artifact Overview + +Running the benchmarks with coverage enabled (`NEO_VM_BENCHMARK=1 NEO_BENCHMARK_COVERAGE=1 dotnet run …`) emits the following files under `BenchmarkArtifacts/`: + +- `opcode-metrics-*.csv`, `syscall-metrics-*.csv`, `native-metrics-*.csv` + - Per-operation measurements for every variant (Baseline, Single, Saturated). + - Columns now include: + - `TotalIterations`, `TotalDataBytes`, `TotalElements` + - `TotalAllocatedBytes`, `AllocatedBytesPerIteration`, `AllocatedBytesPerByte`, `AllocatedBytesPerElement` + - `AverageStackDepth`, `PeakStackDepth`, `AverageAltStackDepth`, `PeakAltStackDepth` + - `TotalGasConsumed`, `GasPerIteration` + +- `benchmark-metrics-*.csv` + - The merged view of all individual metric files. Useful for bulk analysis. + +- `benchmark-scaling.csv` + - Derived ratios comparing Baseline, Single, and Saturated variants: + - Time growth (`BaselineNsPerIteration`, `SingleNsPerIteration`, `SaturatedNsPerIteration`, `IterationScale`). + - Time-per-byte growth (`...NsPerByte`, `ByteScale`). + - Allocation growth (`...AllocatedPerIteration`, `AllocatedIterationScale`, `...AllocatedPerByte`, `AllocatedByteScale`). + - Stack growth (`...AvgStackDepth`, `StackDepthScale`, `...AvgAltStackDepth`, `AltStackDepthScale`). + - Gas growth (`...GasPerIteration`, `GasIterationScale`). + - Also includes the total bytes and total gas consumed per variant for reference. + +Use these ratios to spot super-linear behaviour quickly: + +- **Time scaling** (`IterationScale`, `ByteScale`): values >1 indicate the operation slows down when the workload grows. +- **Allocation scaling**: highlights memory or serialization hotspots that might need higher gas. +- **Stack scaling**: useful for operations that grow recursion or temporary stack usage. +- **Gas scaling**: confirms whether native/syscall implementations are already charging proportionally. + +## Workflow Tips + +1. Run the suite once to produce artifacts. +2. Load `benchmark-scaling.csv` into your analysis tool (Excel, pandas, etc.). +3. Filter by component (`Opcode`, `Syscall`, `NativeContract`) and operation id to inspect ratios. +4. Fit piecewise-linear gas models using the `...PerIteration` and `...PerByte` numbers. +5. Cross-check with `TotalGasConsumed` to ensure the runtime fee aligns with the measured cost. + +## Extensibility + +The recorder can be extended further via `BenchmarkResultRecorder` if additional telemetry is required (e.g., VM reference counts, notification counts). All stack/gas metrics are plumbed through the same APIs used for time and allocation. diff --git a/docs/benchmarking/neo-vm-benchmarking-roadmap.md b/docs/benchmarking/neo-vm-benchmarking-roadmap.md new file mode 100644 index 0000000000..27f40fe2d7 --- /dev/null +++ b/docs/benchmarking/neo-vm-benchmarking-roadmap.md @@ -0,0 +1,97 @@ +# Neo VM Benchmarking Roadmap + +_Last updated: 2025-09-20T14:24:59+08:00_ + +## 1. Objectives +- Provide repeatable, profiled benchmarks for **all VM opcodes**, **system calls**, and **native contract methods** under varying input complexity. +- Produce structured measurement outputs that can drive a **dynamic gas pricing model** based on observed computational cost. +- Deliver documentation and automation so contributors can reproduce results and extend the suite. + +## 2. Scope & Non-Goals +- **In scope:** BenchmarkDotNet-based harnesses, scenario generators, ApplicationEngine-driven syscall/native measurements, data export, CI orchestration. +- **Out of scope (for now):** On-chain deployment utilities, UI visualisations, gas model fitting (handled downstream once data exists). + +## 3. Current State Summary +| Area | Status | Notes | +|------|--------|-------| +| Opcode coverage | In-progress | Systematic registry + coverage CSV for opcodes; ensure enforcement remains pending. | +| Syscalls/native | Stubbed or PoC-only | `BenchmarkEngine` fakes syscalls; native contracts unused. | +| Metrics | Structured CSV | Recorder exports per-scenario metrics with complexity-normalised columns. | +| Input scaling | Single hardcoded cases | Lacks micro/standard/stress comparisons. | +| Documentation | Absent | No contributor guidance. + +## 4. Deliverables Overview +- Shared benchmarking infrastructure (scenario registry, execution engine extensions, profile definitions). +- Opcode, syscall, and native-contract benchmark suites with parameterised workloads. +- Data export pipeline (CSV/JSON) + summary report generation. +- Developer documentation: usage guide, contribution checklist, troubleshooting. +- CI integration for scheduled runs or manual triggers. + +## 5. Work Breakdown Structure + +### Phase 0 – Documentation & Alignment *(current phase)* +- [x] Draft roadmap (this document). +- [ ] Sign-off / capture feedback (update doc as decisions land). + +### Phase 1 – Foundational Infrastructure +- [x] Scenario profile definitions (micro/standard/stress), reusable scenario interface, benchmark suite base class. +- [x] Enhanced `BenchmarkEngine` hooks (pre/post instruction, gas-limit execution with actual measurement, metrics collection in Release). +- [x] Structured result recorder (in-memory model + CSV/JSON writer). + +### Phase 2 – Opcode Coverage +- [ ] Opcode registry mapping → scenario generators per category (push, stack, arithmetic, control, etc.). _Push, stack, arithmetic, bitwise, logic, splice, slot, type, and core control opcodes covered; compound/native interop still pending._ +- [ ] Automated stack priming & cleanup to avoid leaks between iterations. _Implemented for handled categories; extend as new ones arrive._ +- [ ] BenchmarkDotNet suite enumerating all opcodes with baseline/single/saturated runs. +- [ ] Validation scripts to ensure every opcode has a scenario. _Coverage CSV emitted; integrate enforcement once remaining categories land._ + +### Phase 3 – Syscall Benchmarking +- [x] ApplicationEngine harness with seeded snapshot & deterministic environment. _(BenchmarkApplicationEngine establishes a per-run MemoryStore snapshot and timing hooks.)_ +- [ ] Scenario builder reflecting over `InteropService` entries to auto-generate cases. _(Zero-argument runtime syscalls plus log/notify/burngas/getnotifications/currentSigners/checkWitness/loadScript covered; remaining interops (contract/storage) and richer argument synthesis pending.)_ +- [ ] Input scaling (small vs large payloads, varying iterator counts). +- [x] Metrics capture for syscall latency. _(Recorder writes per-scenario syscall/native timings; allocation tracking still TBD.)_ + +### Phase 4 – Native Contract Benchmarking +- [x] Fixture generating canonical blockchain state per native contract (NativeBenchmarkStateFactory provisions a MemoryStore-backed NeoSystem snapshot).. +- [ ] Scenarios for key entry points (transfer, balance, claim, register, etc.). _(Initial read-only calls for Policy/GAS/NEO/Ledger measured; transactional paths pending.)_. +- [ ] Complex workload variants (batch transfers, large storage payloads). + +### Phase 5 – Data Pipeline & Reporting +- [x] Aggregate run output into artefacts (`BenchmarkDotNet` reports + custom summaries). _(CSV summaries emitted for opcode & syscall suites.)_ +- [ ] Provide comparison tooling (e.g., Python/PowerShell script) for historical tracking. +- [ ] Define schema for downstream gas modelling consumers. + +### Phase 6 – Automation & Quality Gates +- [ ] Add CLI command(s) or scripts to run targeted suites. +- [ ] Integrate with CI (smoke subset by default, full suite on-demand or nightly). +- [ ] Add validation tests (e.g., ensure scenario counts match opcode/syscall inventories). + +### Phase 7 – Documentation & Handover +- [ ] Contributor guide covering setup, extending scenarios, interpreting data. +- [ ] Troubleshooting section (common VM setup pitfalls, snapshot regeneration, etc.). +- [ ] Final review & checklist completion. + +## 6. Milestone Tracking +| Milestone | Target | Owner | Status | +|-----------|--------|-------|--------| +| Phase 1 foundation | TBC | _assigned later_ | Not started | +| Phase 2 opcode coverage | TBC | _assigned later_ | In progress | +| Phase 3 syscall coverage | TBC | _assigned later_ | In progress | +| Phase 4 native coverage | TBC | _assigned later_ | In progress | +| Phase 5 pipeline | TBC | _assigned later_ | Not started | +| Phase 6 automation | TBC | _assigned later_ | Not started | +| Phase 7 documentation | TBC | _assigned later_ | Not started | + +## 7. Open Questions / Decisions Needed +1. **Metric granularity:** Do we need allocation/GC metrics alongside CPU time? (Impacts tooling.) +2. **Data retention:** Where should benchmark artefacts be stored (repo vs external storage)? +3. **CI cadence:** Full suite can be long-running; confirm acceptable schedule. +4. **Snapshot seeding:** Agree on baseline chain state for native-contract runs. +5. **Gas-model integration:** Define interface for downstream modelling scripts (JSON schema). + +## 8. Next Steps +1. Gather feedback on this roadmap (update Phase 0 checklist). +2. Kick off Phase 1 tasks (scenario abstractions, engine hooks, metrics recorder). +3. Log progress in this document (update checkboxes, timestamps). + +--- +_This roadmap should be kept alongside implementation PRs. Update phase sections as tasks complete or requirements evolve._