Performance tests with baseline from Pypi (pythonnet#1667)

* reenable perf tests for 3.0
* Get baseline dll from pypi

fixes pythonnet#1368

Co-authored-by: Victor Nova <[email protected]>

Author: filmor

.github/workflows/main.yml

@@ -72,5 +72,10 @@ jobs:
       - name: Python tests run from .NET
         run: dotnet test --runtime any-${{ matrix.platform }} src/python_tests_runner/
 
-      # TODO: Run perf tests
+      - name: Perf tests
+        if: ${{ matrix.python == '3.8' }}
+        run: |
+          pip install --force --no-deps --target src/perf_tests/baseline/ pythonnet==2.5.2
+          dotnet test --configuration Release --runtime any-${{ matrix.platform }} --logger "console;verbosity=detailed" src/perf_tests/
+
       # TODO: Run mono tests on Windows?

.gitignore

@@ -1,4 +1,5 @@
 /src/runtime/interopNative.cs
+/src/perf_tests/baseline/
 
 # General binaries and Build results
 *.dll

src/perf_tests/BaselineComparisonBenchmarkBase.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Reflection;
 
@@ -17,8 +18,7 @@ public BaselineComparisonBenchmarkBase()
             try {
                 PythonEngine.Initialize();
                 Console.WriteLine("Python Initialized");
-                if (PythonEngine.BeginAllowThreads() == IntPtr.Zero)
-                    throw new PythonException();
+                Trace.Assert(PythonEngine.BeginAllowThreads() != IntPtr.Zero);
                 Console.WriteLine("Threading enabled");
             }
             catch (Exception e) {
@@ -28,6 +28,11 @@ public BaselineComparisonBenchmarkBase()
         }
 
         static BaselineComparisonBenchmarkBase()
+        {
+            SetupRuntimeResolve();
+        }
+
+        public static void SetupRuntimeResolve()
         {
             string pythonRuntimeDll = Environment.GetEnvironmentVariable(BaselineComparisonConfig.EnvironmentVariableName);
             if (string.IsNullOrEmpty(pythonRuntimeDll))

src/perf_tests/BaselineComparisonConfig.cs

@@ -5,7 +5,8 @@
 
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Jobs;
-using BenchmarkDotNet.Horology;
+
+using Perfolizer.Horology;
 
 namespace Python.PerformanceTests
 {

src/perf_tests/BenchmarkTests.cs

@@ -30,14 +30,14 @@ public void SetUp()
         public void ReadInt64Property()
         {
             double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports);
-            AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.57);
+            AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 1.35);
         }
 
         [Test]
         public void WriteInt64Property()
         {
             double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports);
-            AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.57);
+            AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 1.25);
         }
 
         static double GetOptimisticPerfRatio(

src/perf_tests/Python.PerformanceTests.csproj

@@ -3,11 +3,25 @@
   <PropertyGroup>
     <TargetFramework>net472</TargetFramework>
     <IsPackable>false</IsPackable>
-    <Platforms>x64;x86</Platforms>
+    <Platforms>x64</Platforms>
+    <PlatformTarget>x64</PlatformTarget>
+  
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" Version="0.12.0" />
+    <Content Include="baseline\Python.Runtime.dll">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
+  <ItemGroup>
+    <Reference Include="baseline/Python.Runtime.dll">
+      <Private>false</Private>
+    </Reference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.1" />
     <PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
@@ -17,23 +31,17 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
-    <PackageReference Include="pythonnet" Version="2.3.0" GeneratePathProperty="true">
-      <IncludeAssets>compile</IncludeAssets>
-    </PackageReference>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
   </ItemGroup>
 
   <Target Name="GetRuntimeLibBuildOutput" BeforeTargets="Build">
-    <MSBuild Projects="..\runtime\Python.Runtime.csproj" Properties="PYTHONNET_PY3_VERSION=PYTHON38;Python3Version=PYTHON38;OutputPath=bin\for_perf\">
+    <MSBuild Projects="..\runtime\Python.Runtime.csproj" Properties="OutputPath=bin\for_perf\;Configuration=Release;TargetFramework=netstandard2.0" Targets="Build">
       <Output TaskParameter="TargetOutputs" ItemName="NewPythonRuntime" />
     </MSBuild>
   </Target>
 
-  <Target Name="CopyBaseline" AfterTargets="Build">
-    <Copy SourceFiles="$(Pkgpythonnet)\lib\net40\Python.Runtime.dll" DestinationFolder="$(OutDir)\baseline" />
-  </Target>
-
   <Target Name="CopyNewBuild" AfterTargets="Build">
+    <Message Text="Outputs: @(NewPythonRuntime)" Importance="high" />
     <Copy SourceFiles="@(NewPythonRuntime)" DestinationFolder="$(OutDir)\new" />
   </Target>
 

src/perf_tests/PythonCallingNetBenchmark.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Reflection;
 using System.Text;
 
 using BenchmarkDotNet.Attributes;
@@ -17,11 +18,11 @@ public void ReadInt64Property()
             {
                 var locals = new PyDict();
                 locals.SetItem("a", new NetObject().ToPython());
-                PythonEngine.Exec($@"
+                Exec($@"
 s = 0
 for i in range(50000):
   s += a.{nameof(NetObject.LongProperty)}
-", locals: locals.Handle);
+", locals: locals);
             }
         }
 
@@ -30,13 +31,24 @@ public void WriteInt64Property() {
             using (Py.GIL()) {
                 var locals = new PyDict();
                 locals.SetItem("a", new NetObject().ToPython());
-                PythonEngine.Exec($@"
+                Exec($@"
 s = 0
 for i in range(50000):
   a.{nameof(NetObject.LongProperty)} += i
-", locals: locals.Handle);
+", locals: locals);
             }
         }
+
+        static void Exec(string code, PyDict locals)
+        {
+            MethodInfo exec = typeof(PythonEngine).GetMethod(nameof(PythonEngine.Exec));
+            object localsArg = typeof(PyObject).Assembly.GetName().Version.Major >= 3
+                ? locals : locals.Handle;
+            exec.Invoke(null, new[]
+            {
+                code, localsArg, null
+            });
+        }
     }
 
     class NetObject