forked from pythonnet/pythonnet
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathBenchmarkTests.cs
More file actions
74 lines (63 loc) · 2.87 KB
/
BenchmarkTests.cs
File metadata and controls
74 lines (63 loc) · 2.87 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Reflection;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Running;
using NUnit.Framework;
namespace Python.PerformanceTests
{
public class BenchmarkTests
{
Summary summary;
[OneTimeSetUp]
public void SetUp()
{
Environment.CurrentDirectory = Path.Combine(DeploymentRoot, "new");
this.summary = BenchmarkRunner.Run<PythonCallingNetBenchmark>();
Assert.IsNotEmpty(this.summary.Reports);
Assert.IsTrue(
condition: this.summary.Reports.All(r => r.Success),
message: "BenchmarkDotNet failed to execute or collect results of performance tests. See logs above.");
}
[Test]
public void ReadInt64Property()
{
double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports);
AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.57);
}
[Test]
public void WriteInt64Property()
{
double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports);
AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.57);
}
static double GetOptimisticPerfRatio(
IReadOnlyList<BenchmarkReport> reports,
[CallerMemberName] string methodName = null)
{
reports = reports.Where(r => r.BenchmarkCase.Descriptor.WorkloadMethod.Name == methodName).ToArray();
if (reports.Count == 0)
throw new ArgumentException(
message: $"No reports found for {methodName}. "
+ "You have to match test method name to benchmark method name or "
+ "pass benchmark method name explicitly",
paramName: nameof(methodName));
var baseline = reports.Single(r => r.BenchmarkCase.Job.ResolvedId == "baseline").ResultStatistics;
var @new = reports.Single(r => r.BenchmarkCase.Job.ResolvedId != "baseline").ResultStatistics;
double newTimeOptimistic = @new.Mean - (@new.StandardDeviation + baseline.StandardDeviation) * 0.5;
return newTimeOptimistic / baseline.Mean;
}
public static string DeploymentRoot => Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
public static void AssertPerformanceIsBetterOrSame(
double actual, double target,
double wiggleRoom = 1.1, [CallerMemberName] string testName = null) {
double threshold = target * wiggleRoom;
Assert.LessOrEqual(actual, threshold,
$"{testName}: {actual:F3} > {threshold:F3} (target: {target:F3})"
+ ": perf result is higher than the failure threshold.");
}
}
}