diff --git a/BENCHMARKS.md b/BENCHMARKS.md new file mode 100644 index 0000000..142ac2b --- /dev/null +++ b/BENCHMARKS.md @@ -0,0 +1,146 @@ +# CMW Benchmarks + +This file contains comprehensive benchmark tests for the CMW (Conceptual Message Wrappers) library performance. + +## Running Benchmarks + +### All Benchmarks +```bash +go test -bench=. -benchmem +``` + +### Specific Benchmark Categories + +#### Marshaling Benchmarks +```bash +# All marshaling benchmarks +go test -bench="BenchmarkCMW_Marshal" -benchmem + +# JSON marshaling only +go test -bench="BenchmarkCMW_MarshalJSON" -benchmem + +# CBOR marshaling only +go test -bench="BenchmarkCMW_MarshalCBOR" -benchmem +``` + +#### Unmarshaling Benchmarks +```bash +# All unmarshaling benchmarks +go test -bench="BenchmarkCMW_Unmarshal" -benchmem + +# JSON unmarshaling only +go test -bench="BenchmarkCMW_UnmarshalJSON" -benchmem + +# CBOR unmarshaling only +go test -bench="BenchmarkCMW_UnmarshalCBOR" -benchmem +``` + +#### X.509 Extension Benchmarks +```bash +# All X.509 extension operations +go test -bench="BenchmarkCMW_.*X509" -benchmem + +# Encoding only +go test -bench="BenchmarkCMW_EncodeX509" -benchmem + +# Decoding only +go test -bench="BenchmarkCMW_DecodeX509" -benchmem +``` + +#### Data Size Comparisons +```bash +# Small data benchmarks +go test -bench=".*Small.*" -benchmem + +# Medium data benchmarks +go test -bench=".*Medium.*" -benchmem + +# Large data benchmarks +go test -bench=".*Large.*" -benchmem + +# Collection benchmarks +go test -bench=".*Collection.*" -benchmem +``` + +#### Format-Specific Benchmarks +```bash +# CBOR Tag format benchmarks +go test -bench=".*Tag.*" -benchmem + +# Auto-detection benchmarks +go test -bench="BenchmarkCMW_Deserialize" -benchmem + +# Format sniffing benchmarks +go test -bench="BenchmarkSniff" -benchmem +``` + +### Advanced Options + +#### Multiple Runs for Statistical Accuracy +```bash +go test -bench=. -benchmem -count=5 +``` + +#### CPU Profiling +```bash +go test -bench=. -benchmem -cpuprofile=cpu.prof +``` + +#### Memory Profiling +```bash +go test -bench=. -benchmem -memprofile=mem.prof +``` + +#### Long-Running Benchmarks +```bash +go test -bench=. -benchmem -benchtime=10s +``` + +## Benchmark Data Sizes + +The benchmarks test with different data sizes to understand performance characteristics: + +- **Small**: ~15 bytes - Typical for simple attestation tokens +- **Medium**: ~1KB - Typical for complex attestation evidence +- **Large**: ~100KB - Large attestation bundles or collections +- **Simple Collection**: 2 items - Basic collection structure +- **Complex Collection**: 50 nested items - Realistic nested attestation data + +## Performance Insights + +Based on the benchmark results: + +1. **CBOR vs JSON**: CBOR is generally faster and uses less memory than JSON +2. **CBOR Tag Format**: Much faster than CBOR record format for monads +3. **Collections**: Performance scales reasonably with collection size +4. **X.509 Extensions**: CBOR encoding is faster than JSON for extensions +5. **Auto-detection**: Small overhead for format detection + +## Adding New Benchmarks + +When adding new benchmarks: + +1. Follow the naming convention: `BenchmarkCMW___` +2. Include `b.ReportAllocs()` to track memory allocations +3. Use `b.ResetTimer()` after setup code +4. Handle errors appropriately with `b.Fatal(err)` +5. Create realistic test data that represents actual use cases + +Example: +```go +func BenchmarkCMW_NewOperation_TestData(b *testing.B) { + // Setup code here + testData := setupTestData() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + result, err := cmw.NewOperation(testData) + if err != nil { + b.Fatal(err) + } + _ = result // Prevent compiler optimization + } +} +``` \ No newline at end of file diff --git a/benchmark_test.go b/benchmark_test.go new file mode 100644 index 0000000..5bf8bda --- /dev/null +++ b/benchmark_test.go @@ -0,0 +1,748 @@ +// Copyright 2023 Contributors to the Veraison project. +// SPDX-License-Identifier: Apache-2.0 + +package cmw + +import ( + "testing" +) + +// Benchmark data generators for different sizes and complexities + +func makeSmallMonad() *CMW { + cmw, _ := NewMonad("application/json", []byte("small test data"), ReferenceValues) + return cmw +} + +func makeMediumMonad() *CMW { + // Create ~1KB of data + largeData := make([]byte, 1024) + for i := range largeData { + largeData[i] = byte(i % 256) + } + cmw, _ := NewMonad("application/cbor", largeData, ReferenceValues, Endorsements) + return cmw +} + +func makeLargeMonad() *CMW { + // Create ~100KB of data + largeData := make([]byte, 100*1024) + for i := range largeData { + largeData[i] = byte(i % 256) + } + cmw, _ := NewMonad("application/json", largeData, ReferenceValues, Endorsements, Evidence) + return cmw +} + +func makeSimpleCollection() *CMW { + root, _ := NewCollection("tag:example.com,2024:simple") + + node1, _ := NewMonad("application/json", []byte("data1"), ReferenceValues) + _ = root.AddCollectionItem("item1", node1) + + node2, _ := NewMonad("application/cbor", []byte("data2"), Endorsements) + _ = root.AddCollectionItem("item2", node2) + + return root +} + +func makeComplexCollection() *CMW { + // Create a complex nested collection similar to makeCMWCollection but more complex + root, _ := NewCollection("tag:ietf.org,2024:benchmark") + + // Add multiple nested collections + for i := 0; i < 5; i++ { + sub, _ := NewCollection("tag:ietf.org,2024:sub") + + // Add multiple items to each sub-collection + for j := 0; j < 10; j++ { + data := make([]byte, 100+j*10) // Variable sized data + for k := range data { + data[k] = byte((i*10 + j + k) % 256) + } + + node, _ := NewMonad("application/cbor", data, ReferenceValues, Endorsements) + _ = sub.AddCollectionItem(j, node) + } + + _ = root.AddCollectionItem(i, sub) + } + + return root +} + +// JSON Marshaling Benchmarks + +func BenchmarkCMW_MarshalJSON_SmallMonad(b *testing.B) { + cmw := makeSmallMonad() + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := cmw.MarshalJSON() + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_MarshalJSON_MediumMonad(b *testing.B) { + cmw := makeMediumMonad() + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := cmw.MarshalJSON() + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_MarshalJSON_LargeMonad(b *testing.B) { + cmw := makeLargeMonad() + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := cmw.MarshalJSON() + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_MarshalJSON_SimpleCollection(b *testing.B) { + cmw := makeSimpleCollection() + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := cmw.MarshalJSON() + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_MarshalJSON_ComplexCollection(b *testing.B) { + cmw := makeComplexCollection() + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := cmw.MarshalJSON() + if err != nil { + b.Fatal(err) + } + } +} + +// CBOR Marshaling Benchmarks + +func BenchmarkCMW_MarshalCBOR_SmallMonad(b *testing.B) { + cmw := makeSmallMonad() + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := cmw.MarshalCBOR() + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_MarshalCBOR_MediumMonad(b *testing.B) { + cmw := makeMediumMonad() + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := cmw.MarshalCBOR() + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_MarshalCBOR_LargeMonad(b *testing.B) { + cmw := makeLargeMonad() + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := cmw.MarshalCBOR() + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_MarshalCBOR_SimpleCollection(b *testing.B) { + cmw := makeSimpleCollection() + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := cmw.MarshalCBOR() + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_MarshalCBOR_ComplexCollection(b *testing.B) { + cmw := makeComplexCollection() + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := cmw.MarshalCBOR() + if err != nil { + b.Fatal(err) + } + } +} + +// JSON Unmarshaling Benchmarks + +func BenchmarkCMW_UnmarshalJSON_SmallMonad(b *testing.B) { + cmw := makeSmallMonad() + jsonData, _ := cmw.MarshalJSON() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var newCMW CMW + err := newCMW.UnmarshalJSON(jsonData) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_UnmarshalJSON_MediumMonad(b *testing.B) { + cmw := makeMediumMonad() + jsonData, _ := cmw.MarshalJSON() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var newCMW CMW + err := newCMW.UnmarshalJSON(jsonData) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_UnmarshalJSON_LargeMonad(b *testing.B) { + cmw := makeLargeMonad() + jsonData, _ := cmw.MarshalJSON() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var newCMW CMW + err := newCMW.UnmarshalJSON(jsonData) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_UnmarshalJSON_SimpleCollection(b *testing.B) { + cmw := makeSimpleCollection() + jsonData, _ := cmw.MarshalJSON() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var newCMW CMW + err := newCMW.UnmarshalJSON(jsonData) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_UnmarshalJSON_ComplexCollection(b *testing.B) { + cmw := makeComplexCollection() + jsonData, _ := cmw.MarshalJSON() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var newCMW CMW + err := newCMW.UnmarshalJSON(jsonData) + if err != nil { + b.Fatal(err) + } + } +} + +// CBOR Unmarshaling Benchmarks + +func BenchmarkCMW_UnmarshalCBOR_SmallMonad(b *testing.B) { + cmw := makeSmallMonad() + cborData, _ := cmw.MarshalCBOR() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var newCMW CMW + err := newCMW.UnmarshalCBOR(cborData) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_UnmarshalCBOR_MediumMonad(b *testing.B) { + cmw := makeMediumMonad() + cborData, _ := cmw.MarshalCBOR() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var newCMW CMW + err := newCMW.UnmarshalCBOR(cborData) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_UnmarshalCBOR_LargeMonad(b *testing.B) { + cmw := makeLargeMonad() + cborData, _ := cmw.MarshalCBOR() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var newCMW CMW + err := newCMW.UnmarshalCBOR(cborData) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_UnmarshalCBOR_SimpleCollection(b *testing.B) { + cmw := makeSimpleCollection() + cborData, _ := cmw.MarshalCBOR() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var newCMW CMW + err := newCMW.UnmarshalCBOR(cborData) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_UnmarshalCBOR_ComplexCollection(b *testing.B) { + cmw := makeComplexCollection() + cborData, _ := cmw.MarshalCBOR() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var newCMW CMW + err := newCMW.UnmarshalCBOR(cborData) + if err != nil { + b.Fatal(err) + } + } +} + +// Auto-detection Deserialize Benchmarks + +func BenchmarkCMW_Deserialize_JSON_SmallMonad(b *testing.B) { + cmw := makeSmallMonad() + jsonData, _ := cmw.MarshalJSON() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var newCMW CMW + err := newCMW.Deserialize(jsonData) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_Deserialize_CBOR_SmallMonad(b *testing.B) { + cmw := makeSmallMonad() + cborData, _ := cmw.MarshalCBOR() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var newCMW CMW + err := newCMW.Deserialize(cborData) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_Deserialize_JSON_ComplexCollection(b *testing.B) { + cmw := makeComplexCollection() + jsonData, _ := cmw.MarshalJSON() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var newCMW CMW + err := newCMW.Deserialize(jsonData) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_Deserialize_CBOR_ComplexCollection(b *testing.B) { + cmw := makeComplexCollection() + cborData, _ := cmw.MarshalCBOR() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var newCMW CMW + err := newCMW.Deserialize(cborData) + if err != nil { + b.Fatal(err) + } + } +} + +// Format Detection Benchmarks + +func BenchmarkSniff_JSON_Record(b *testing.B) { + cmw := makeSmallMonad() + jsonData, _ := cmw.MarshalJSON() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _ = Sniff(jsonData) + } +} + +func BenchmarkSniff_CBOR_Record(b *testing.B) { + cmw := makeSmallMonad() + cborData, _ := cmw.MarshalCBOR() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _ = Sniff(cborData) + } +} + +func BenchmarkSniff_JSON_Collection(b *testing.B) { + cmw := makeSimpleCollection() + jsonData, _ := cmw.MarshalJSON() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _ = Sniff(jsonData) + } +} + +func BenchmarkSniff_CBOR_Collection(b *testing.B) { + cmw := makeSimpleCollection() + cborData, _ := cmw.MarshalCBOR() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _ = Sniff(cborData) + } +} + +// X.509 Extension Benchmarks + +func BenchmarkCMW_EncodeX509Extension_JSON_SimpleCollection(b *testing.B) { + cmw := makeSimpleCollection() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := cmw.EncodeX509Extension(ChoiceJson, false) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_EncodeX509Extension_CBOR_SimpleCollection(b *testing.B) { + cmw := makeSimpleCollection() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := cmw.EncodeX509Extension(ChoiceCbor, false) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_EncodeX509Extension_JSON_ComplexCollection(b *testing.B) { + cmw := makeComplexCollection() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := cmw.EncodeX509Extension(ChoiceJson, false) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_EncodeX509Extension_CBOR_ComplexCollection(b *testing.B) { + cmw := makeComplexCollection() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := cmw.EncodeX509Extension(ChoiceCbor, false) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_DecodeX509Extension_JSON_SimpleCollection(b *testing.B) { + cmw := makeSimpleCollection() + ext, _ := cmw.EncodeX509Extension(ChoiceJson, false) + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := DecodeX509Extension(*ext) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_DecodeX509Extension_CBOR_SimpleCollection(b *testing.B) { + cmw := makeSimpleCollection() + ext, _ := cmw.EncodeX509Extension(ChoiceCbor, false) + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := DecodeX509Extension(*ext) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_DecodeX509Extension_JSON_ComplexCollection(b *testing.B) { + cmw := makeComplexCollection() + ext, _ := cmw.EncodeX509Extension(ChoiceJson, false) + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := DecodeX509Extension(*ext) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_DecodeX509Extension_CBOR_ComplexCollection(b *testing.B) { + cmw := makeComplexCollection() + ext, _ := cmw.EncodeX509Extension(ChoiceCbor, false) + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := DecodeX509Extension(*ext) + if err != nil { + b.Fatal(err) + } + } +} + +// CBOR Tag Format Benchmarks + +func BenchmarkCMW_MarshalCBOR_Tag_SmallMonad(b *testing.B) { + cmw := makeSmallMonad() + cmw.UseCBORTagFormat() // Switch to tag format + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := cmw.MarshalCBOR() + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_MarshalCBOR_Tag_MediumMonad(b *testing.B) { + cmw := makeMediumMonad() + cmw.UseCBORTagFormat() // Switch to tag format + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := cmw.MarshalCBOR() + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_UnmarshalCBOR_Tag_SmallMonad(b *testing.B) { + cmw := makeSmallMonad() + cmw.UseCBORTagFormat() + cborData, _ := cmw.MarshalCBOR() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var newCMW CMW + err := newCMW.UnmarshalCBOR(cborData) + if err != nil { + b.Fatal(err) + } + } +} + +// Collection Operations Benchmarks + +func BenchmarkCMW_GetCollectionItem(b *testing.B) { + cmw := makeComplexCollection() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + // Access different items to avoid caching effects + key := i % 5 + _, err := cmw.GetCollectionItem(key) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_GetCollectionMeta(b *testing.B) { + cmw := makeComplexCollection() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := cmw.GetCollectionMeta() + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCMW_AddCollectionItem(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + b.StopTimer() + root, _ := NewCollection("tag:ietf.org,2024:benchmark") + node, _ := NewMonad("application/json", []byte("test data"), ReferenceValues) + b.StartTimer() + + err := root.AddCollectionItem(i, node) + if err != nil { + b.Fatal(err) + } + } +} + +// Type and Value Benchmarks + +func BenchmarkType_Set_String(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var t Type + err := t.Set("application/json") + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkType_Set_Uint16(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var t Type + err := t.Set(uint16(50)) // application/json + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkValue_Set(b *testing.B) { + data := make([]byte, 1024) + for i := range data { + data[i] = byte(i % 256) + } + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var v Value + err := v.Set(data) + if err != nil { + b.Fatal(err) + } + } +} \ No newline at end of file