Merge pull request #9 from cryspen/wysiwys/parse-metadata

Use new metadata format

Author: wysiwys

.github/workflows/build-and-test.yml

@@ -19,24 +19,24 @@ jobs:
       - uses: ./ # local action in repo
         with:
           tool: 'cargo'
-          platform: 'default'
+          os: 'ubuntu-latest'
           output-file-path: ./test/data/extract/cargo_output.txt
           data-out-path: output.txt
       - uses: ./ # local action in repo
         with:
           tool: 'cargo'
-          platform: 'default'
+          os: 'ubuntu-latest'
           output-file-path: ./test/data/extract/cargo_output2.txt
           data-out-path: output.txt
       - uses: ./ # local action in repo
         with:
           tool: 'cargo'
-          platform: 'default'
+          os: 'ubuntu-latest'
           output-file-path: ./test/data/extract/cargo_output3.txt
           data-out-path: output.txt
       - uses: ./ # local action in repo
         with:
           tool: 'cargo'
-          platform: 'default'
+          os: 'ubuntu-latest'
           output-file-path: ./test/data/extract/criterion_output.txt
           data-out-path: output.txt

.github/workflows/minimal.yml

@@ -16,6 +16,6 @@ jobs:
       - uses: ./ # local action in repo
         with:
           tool: 'cargo'
-          platform: 'default'
+          os: 'ubuntu-latest'
           output-file-path: ./test/data/extract/cargo_output.txt
           data-out-path: output.txt

README.md

@@ -9,21 +9,29 @@ Currently supported sources:
 ## Fields
 - `name` (required): name of the benchmark
 - `tool` (required): tool used to get benchmark output. One of `["cargo"]`
-- `platform` (required): a string describing the platform
+- `os` (required): a string describing the os
 - `output-file-path` (required): a path to a file containing the output of the benchmark tool
 - `data-out-path` (required): the path where the output of the action should be written
 
+## Metadata format
+
+The benchmark name in the `cargo` benchmarks can be provided as a `/`-separated string with the format `category/key size/name/platform/api`. The key size should be an integer. Some fields in this string can be left blank. Any unspecified or invalid fields will be parsed to `undefined`.
+
 ## Output data format
 
 The output will be written to `data-out-path` in a standardized JSON format:
 ```json
 [
-    {
-        "name": "My Custom Smaller Is Better Benchmark - Memory Used",
-        "unit": "Megabytes",
-        "platform": "ubuntu-latest",
-        "value": 100,
-        "range": "3",
-    }
+  {
+    "api": "unpacked",
+    "category": "ML-KEM",
+    "keySize": 768,
+    "name": "PK Validation",
+    "os": "ubuntu-latest",
+    "platform": "neon",
+    "range": "± 123",
+    "unit": "ns/iter",
+    "value": 12314,
+  },
 ]
 ```

action-types.yml

@@ -5,7 +5,7 @@ inputs:
     type: enum
     allowed-values:
       - cargo
-  platform:
+  os:
     type: string
   output-file-path:
     type: string

action.yml

@@ -13,8 +13,8 @@ inputs:
   tool:
     description: 'Tool to use to get benchmark output. One of "cargo"...'
     required: true
-  platform:
-    description: 'A string describing the platform'
+  os:
+    description: 'A string describing the os'
     required: true
   output-file-path:
     description: 'A path to file which contains the benchmark output'

src/config.ts

@@ -6,7 +6,7 @@ import * as path from 'path';
 export type ToolType = typeof VALID_TOOLS[number];
 export interface Config {
     name: string;
-    platform: string;
+    os: string;
     tool: ToolType;
     outputFilePath: string;
     dataOutPath: string;
@@ -71,22 +71,22 @@ function validateName(name: string) {
     }
     throw new Error('Name must not be empty');
 }
-function validatePlatform(platform: string) {
-    if (platform) {
+function validateOs(os: string) {
+    if (os) {
         return;
     }
-    throw new Error('Platform must not be empty');
+    throw new Error('Os must not be empty');
 }
 
 export async function configFromJobInput(): Promise<Config> {
     const tool: string = core.getInput('tool');
     let outputFilePath: string = core.getInput('output-file-path');
     let dataOutPath: string = core.getInput('data-out-path');
     const name: string = core.getInput('name');
-    const platform: string = core.getInput('platform');
+    const os: string = core.getInput('os');
 
     validateName(name);
-    validatePlatform(platform);
+    validateOs(os);
     validateToolType(tool);
     outputFilePath = await validateOutputFilePath(outputFilePath);
     dataOutPath = await validateDataOutPath(dataOutPath);
@@ -96,6 +96,6 @@ export async function configFromJobInput(): Promise<Config> {
         tool,
         outputFilePath,
         dataOutPath,
-        platform,
+        os,
     };
 }

src/extract.ts

@@ -3,12 +3,61 @@ import { promises as fs } from 'fs';
 import { Config } from './config';
 
 export interface BenchmarkResult {
-    name: string;
     value: number;
     range?: string;
     unit: string;
     extra?: string;
-    platform: string;
+    os: string;
+
+    // from NameMetadata
+    category: string | undefined;
+    keySize: number | undefined;
+    name: string;
+    platform: string | undefined;
+    api: string | undefined;
+}
+
+interface NameMetadata {
+    category: string | undefined;
+    keySize: number | undefined;
+    name: string;
+    platform: string | undefined;
+    api: string | undefined;
+}
+function extractMetadataFromName(name_string: string): NameMetadata {
+    // split by separator
+    const values = name_string.split('/');
+
+    // if only one arg provided, just return name
+    if (values.length === 1) {
+        const name = name_string;
+        return { name, keySize: undefined, category: undefined, platform: undefined, api: undefined };
+    }
+
+    // extract by position
+    const category = values[0] === '' ? undefined : values[0];
+
+    // If keySize not a number, use `undefined`
+    const keySizeParsed = parseInt(values[1]);
+
+    const keySize = isNaN(keySizeParsed) ? undefined : keySizeParsed;
+
+    // if name is not defined, keep entire name_string as name
+    let name = values[2];
+    if (name === undefined || name === '') {
+        name = name_string;
+    }
+
+    const platform = values[3] === '' ? undefined : values[3];
+    const api = values[4] === '' ? undefined : values[4];
+
+    return {
+        category,
+        keySize,
+        name,
+        platform,
+        api,
+    };
 }
 
 function extractCargoResult(config: Config, output: string): BenchmarkResult[] {
@@ -23,17 +72,24 @@ function extractCargoResult(config: Config, output: string): BenchmarkResult[] {
             continue;
         }
 
-        const name = m[1].trim();
+        const name_string = m[1].trim();
         const value = parseFloat(m[2].replace(reComma, ''));
         const unit = m[3].trim();
         const range = m[4].replace(reComma, '');
 
+        // TODO: error handling
+        const { category, keySize, name, platform, api } = extractMetadataFromName(name_string);
+
         ret.push({
-            name,
             value,
             range: `± ${range}`,
             unit: unit,
-            platform: config.platform,
+            os: config.os,
+            category,
+            keySize,
+            name,
+            platform,
+            api,
         });
     }