Merge pull request #14 from cryspen/wysiwys/group-separator

Allow Criterion group separator in name

Author: wysiwys

README.md

@@ -15,7 +15,7 @@ Currently supported sources:
 
 ## Metadata format
 
-The benchmark name in the `cargo` benchmarks can be provided as a `,`-separated string where each item represents a key-value pair with the format `key=value`, e.g. `category=ML-KEM,keySize=44,name=PK Validation,platform=neon,api=unpacked`.
+The benchmark name in the `cargo` benchmarks can be provided as a `,`-separated string where each item represents a key-value pair with the format `key=value`, e.g. `category=ML-KEM,keySize=44,name=PK Validation,platform=neon,api=unpacked`. Additionally, one of these separators may be a `/`, where the left-hand side represents the keys configured for a Criterion group, and the right-hand side represents the keys for a benchmark function on that group, e.g. `category=ML-KEM,keySize=44,name=PK Validation/platform=neon,api=unpacked`. Aside from this separator, no other `/` characters should be included in the name.
 
 Alternatively, the benchmark name can be provided as a simple string, which will then be set as the value of the `name` key. This will only happen if there are no `key=value` pairs found in the benchmark name.
 

scripts/prepare-release.sh

@@ -50,7 +50,7 @@ cp -R node_modules .release/node_modules
 
 git checkout "$version"
 git pull
-git rm -rf node_modules
+git rm -rf --ignore-unmatch node_modules
 rm -rf node_modules  # remove node_modules/.cache
 
 rm -rf dist

src/extract.ts

@@ -14,8 +14,14 @@ export interface BenchmarkResult {
 }
 
 function addNameMetadataToResult(result: BenchmarkResult, nameString: string) {
-    // split by separator
-    const keyValuePairs = nameString.split(',');
+    // replace first '/'
+    // only one is allowed, as a Criterion group/function separator
+    const nameStringProcessed = nameString.replace('/', ',');
+
+    if (nameStringProcessed.includes('/')) {
+        throw new Error("Only one '/' is allowed as a group/function separator");
+    }
+    const keyValuePairs = nameStringProcessed.split(',');
 
     let foundAtLeastOne = false;
 

test/data/extract/criterion_invalid.txt

@@ -0,0 +1,14 @@
+WARNING: HTML report generation will become a non-default optional feature in Criterion.rs 0.4.0.
+This feature is being moved to cargo-criterion (https://github.com/bheisler/cargo-criterion) and will be optional in a future version of Criterion.rs. To silence this warning, either switch to cargo-criterion or enable the 'html_reports' feature in your Cargo.toml.
+
+Gnuplot not found, using plotters backend
+test category=ML-KEM,keySize=1024,name=PK Validation ... bench:         329 ns/iter (+/- 4)
+
+test category=ML-KEM,keySize=512/name=PK Validation/platform=neon,api=unpacked ... bench:        3268 ns/iter (+/- 47)
+
+test category=ML-KEM,keySize=768,name=PK Validation/platform=neon,api=unpacked ... bench:       12314 ns/iter (+/- 123)
+
+
+Warning: Unable to complete 100 samples in 5.0s. You may wish to increase target time to 8.5s, enable flat sampling, or reduce sample count to 50.
+test category=ML-KEM,keySize=512,name=PK Validation,platform=neon,api=unpacked (external random) ... bench:     1672496 ns/iter (+/- 10166)
+

test/data/extract/criterion_output_with_metadata_format.txt

@@ -4,9 +4,9 @@ This feature is being moved to cargo-criterion (https://github.com/bheisler/carg
 Gnuplot not found, using plotters backend
 test category=ML-KEM,keySize=1024,name=PK Validation ... bench:         329 ns/iter (+/- 4)
 
-test category=ML-KEM,keySize=512,name=PK Validation,platform=neon,api=unpacked ... bench:        3268 ns/iter (+/- 47)
+test category=ML-KEM,keySize=512,name=PK Validation/platform=neon,api=unpacked ... bench:        3268 ns/iter (+/- 47)
 
-test category=ML-KEM,keySize=768,name=PK Validation,platform=neon,api=unpacked ... bench:       12314 ns/iter (+/- 123)
+test category=ML-KEM,keySize=768,name=PK Validation/platform=neon,api=unpacked ... bench:       12314 ns/iter (+/- 123)
 
 
 Warning: Unable to complete 100 samples in 5.0s. You may wish to increase target time to 8.5s, enable flat sampling, or reduce sample count to 50.

test/extract.spec.ts

@@ -82,6 +82,15 @@ describe('extractData()', function () {
         } as Config;
         await A.rejects(extractData(config), /^Error: No benchmark result was found in /);
     });
+
+    it('raises an error when more than one / separator', async function () {
+        const config = {
+            os: 'ubuntu-latest',
+            tool: 'cargo',
+            outputFilePath: path.join(__dirname, 'data', 'extract', 'criterion_invalid.txt'),
+        } as Config;
+        await A.rejects(extractData(config), /^Error: Only one '\/' is allowed as a group\/function separator/);
+    });
 });
 
 describe('localWriteBenchmark()', function () {