From 7a8e7f0eccb1b85d05d19add30cd93539ef954c7 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 5 Feb 2025 04:30:17 +0700 Subject: [PATCH 01/18] feat: add swissmap Signed-off-by: Dwi Siswanto --- go.mod | 1 + go.sum | 2 + swissmap/swissmap.go | 188 ++++++++++++++++++++++++++++++++ swissmap/swissmap_bench_test.go | 127 +++++++++++++++++++++ swissmap/swissmap_test.go | 124 +++++++++++++++++++++ 5 files changed, 442 insertions(+) create mode 100644 swissmap/swissmap.go create mode 100644 swissmap/swissmap_bench_test.go create mode 100644 swissmap/swissmap_test.go diff --git a/go.mod b/go.mod index 1b0ae50..8c63baa 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( github.com/charmbracelet/lipgloss v0.13.0 // indirect github.com/charmbracelet/x/ansi v0.3.2 // indirect github.com/cloudflare/circl v1.3.7 // indirect + github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.11.4 // indirect github.com/fatih/color v1.15.0 // indirect diff --git a/go.sum b/go.sum index 3d3658b..e92e4af 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,8 @@ github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObk github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 h1:Ew0znI2JatzKy52N1iS5muUsHkf2UJuhocH7uFW7jjs= +github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/swissmap/swissmap.go b/swissmap/swissmap.go new file mode 100644 index 0000000..32063a8 --- /dev/null +++ b/swissmap/swissmap.go @@ -0,0 +1,188 @@ +package swissmap + +import ( + "sync" + + "github.com/cockroachdb/swiss" +) + +// Option represents a configuration option for the Map +type Option[K, V comparable] func(*Map[K, V]) + +// WithCapacity sets the initial capacity of the map +func WithCapacity[K, V comparable](capacity int) Option[K, V] { + return func(m *Map[K, V]) { + m.data.Init(capacity) + } +} + +// WithThreadSafety enables thread-safety for the map +func WithThreadSafety[K, V comparable]() Option[K, V] { + return func(m *Map[K, V]) { + m.threadSafe = true + } +} + +// Map is a generic map implementation using swiss.Map with optional thread-safety +type Map[K, V comparable] struct { + mutex sync.RWMutex + threadSafe bool + data *swiss.Map[K, V] +} + +// New creates a new Map with the given options +func New[K, V comparable](options ...Option[K, V]) *Map[K, V] { + m := &Map[K, V]{data: swiss.New[K, V](0)} + + for _, opt := range options { + opt(m) + } + + return m +} + +// lock conditionally acquires the read lock if thread-safety is enabled +func (m *Map[K, V]) lock() { + if m.threadSafe { + m.mutex.Lock() + } +} + +// unlock conditionally releases the read lock if thread-safety is enabled +func (m *Map[K, V]) unlock() { + if m.threadSafe { + m.mutex.Unlock() + } +} + +// rLock conditionally acquires the read lock if thread-safety is enabled +func (m *Map[K, V]) rLock() { + if m.threadSafe { + m.mutex.RLock() + } +} + +// rUnlock conditionally releases the read lock if thread-safety is enabled +func (m *Map[K, V]) rUnlock() { + if m.threadSafe { + m.mutex.RUnlock() + } +} + +// Clear removes all elements from the map +func (m *Map[K, V]) Clear() bool { + m.lock() + defer m.unlock() + + hadElements := m.data.Len() > 0 + m.data.Clear() + return hadElements +} + +// Clone returns a new Map with a copy of the underlying data +func (m *Map[K, V]) Clone() *Map[K, V] { + m.rLock() + defer m.rUnlock() + + clone := New[K, V]() + m.data.All(func(key K, value V) bool { + clone.data.Put(key, value) + + return true + }) + + return clone +} + +// Get retrieves a value from the map +func (m *Map[K, V]) Get(key K) (V, bool) { + m.rLock() + defer m.rUnlock() + + return m.data.Get(key) +} + +// GetKeyWithValue retrieves the first key associated with the given value +func (m *Map[K, V]) GetKeyWithValue(value V) (K, bool) { + m.rLock() + defer m.rUnlock() + + var foundKey K + var found bool + + m.data.All(func(key K, v V) bool { + if v == value { + foundKey = key + found = true + + return false // stop iteration + } + + return true + }) + + return foundKey, found +} + +// GetKeys returns values for the given keys +func (m *Map[K, V]) GetKeys(keys ...K) []V { + m.rLock() + defer m.rUnlock() + + result := make([]V, 0, len(keys)) + for _, key := range keys { + if val, ok := m.data.Get(key); ok { + result = append(result, val) + } + } + + return result +} + +// GetOrDefault returns the value for key or defaultValue if key is not found +func (m *Map[K, V]) GetOrDefault(key K, defaultValue V) V { + m.rLock() + defer m.rUnlock() + + if val, ok := m.data.Get(key); ok { + return val + } + + return defaultValue +} + +// Has checks if a key exists in the map +func (m *Map[K, V]) Has(key K) bool { + m.rLock() + defer m.rUnlock() + + _, ok := m.data.Get(key) + + return ok +} + +// IsEmpty returns true if the map contains no elements +func (m *Map[K, V]) IsEmpty() bool { + m.rLock() + defer m.rUnlock() + + return m.data.Len() == 0 +} + +// Merge adds all key/value pairs from the input map +func (m *Map[K, V]) Merge(n map[K]V) { + m.lock() + defer m.unlock() + + for k, v := range n { + m.data.Put(k, v) + } +} + +// Set inserts or updates a key/value pair +func (m *Map[K, V]) Set(key K, value V) { + m.lock() + defer m.unlock() + + m.data.Put(key, value) +} diff --git a/swissmap/swissmap_bench_test.go b/swissmap/swissmap_bench_test.go new file mode 100644 index 0000000..b6d19ee --- /dev/null +++ b/swissmap/swissmap_bench_test.go @@ -0,0 +1,127 @@ +package swissmap + +import ( + "fmt" + "testing" +) + +var benchNumItems = []int{1000, 5000, 10_000, 100_000, 250_000, 500_000, 1_000_000} + +func createMaps[K, V comparable](numItems int, threadSafe bool) *Map[K, V] { + options := []Option[K, V]{ + WithCapacity[K, V](numItems), + } + + if threadSafe { + options = append(options, WithThreadSafety[K, V]()) + } + + return New[K, V](options...) +} + +func BenchmarkGet(b *testing.B) { + for _, numItems := range benchNumItems { + b.Run(fmt.Sprintf("items=%d", numItems), func(b *testing.B) { + m := createMaps[string, int](numItems, false) + for i := 0; i < numItems; i++ { + m.Set(fmt.Sprint(i), i) + } + + b.ResetTimer() + b.Run("seq", func(b *testing.B) { + for i := 0; i < b.N; i++ { + m.Get(fmt.Sprint(i % numItems)) + } + }) + + b.ResetTimer() + b.Run("parallel", func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + _, _ = m.Get(fmt.Sprint(i % numItems)) + i++ + } + }) + }) + + b.Run("WithThreadSafety", func(b *testing.B) { + m := createMaps[string, int](numItems, true) + for i := 0; i < numItems; i++ { + m.Set(fmt.Sprint(i), i) + } + + b.ResetTimer() + b.Run("seq", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, _ = m.Get(fmt.Sprint(i % numItems)) + } + }) + + b.ResetTimer() + b.Run("parallel", func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + _, _ = m.Get(fmt.Sprint(i % numItems)) + i++ + } + }) + }) + }) + }) + } +} + +func BenchmarkSet(b *testing.B) { + for _, numItems := range benchNumItems { + b.Run(fmt.Sprintf("items=%d", numItems), func(b *testing.B) { + m := createMaps[string, int](numItems, false) + + b.ResetTimer() + b.Run("seq", func(b *testing.B) { + for i := 0; i < b.N; i++ { + m.Set(fmt.Sprint(i), i) + } + }) + + m.Clear() + + b.ResetTimer() + b.Run("parallel", func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + m.Set(fmt.Sprint(i), i) + i++ + } + }) + }) + + b.Run("WithThreadSafety", func(b *testing.B) { + m := createMaps[string, int](numItems, true) + + b.ResetTimer() + b.Run("seq", func(b *testing.B) { + for i := 0; i < b.N; i++ { + m.Set(fmt.Sprint(i), i) + } + }) + + m.Clear() + + b.ResetTimer() + b.Run("parallel", func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + m.Set(fmt.Sprint(i), i) + i++ + } + }) + }) + }) + + }) + } +} diff --git a/swissmap/swissmap_test.go b/swissmap/swissmap_test.go new file mode 100644 index 0000000..3ab37e2 --- /dev/null +++ b/swissmap/swissmap_test.go @@ -0,0 +1,124 @@ +package swissmap + +import ( + "sync" + "testing" +) + +func TestMap(t *testing.T) { + t.Run("Basic operations", func(t *testing.T) { + m := New[string, int]() + + // Test Set and Get + m.Set("one", 1) + if val, ok := m.Get("one"); !ok || val != 1 { + t.Errorf("expected Get(\"one\") = (1, true), got (%v, %v)", val, ok) + } + + // Test Has + if !m.Has("one") { + t.Error("Has(\"one\") should return true") + } + + // Test GetOrDefault + if val := m.GetOrDefault("two", 2); val != 2 { + t.Errorf("expected GetOrDefault(\"two\", 2) = 2, got %v", val) + } + + // Test IsEmpty + if m.IsEmpty() { + t.Error("IsEmpty() should return false") + } + + // Test Clear + if !m.Clear() { + t.Error("Clear() should return true for non-empty map") + } + if !m.IsEmpty() { + t.Error("map should be empty after Clear()") + } + }) + + t.Run("Concurrent operations", func(t *testing.T) { + m := New[int, int]() + var wg sync.WaitGroup + n := 1000 + + // Concurrent writers + for i := 0; i < n; i++ { + wg.Add(1) + go func(val int) { + defer wg.Done() + m.Set(val, val*2) + }(i) + } + + // Concurrent readers + for i := 0; i < n; i++ { + wg.Add(1) + go func(val int) { + defer wg.Done() + m.Has(val) + m.Get(val) + }(i) + } + + wg.Wait() + + // Verify results + count := 0 + for i := 0; i < n; i++ { + if val, ok := m.Get(i); ok && val == i*2 { + count++ + } + } + if count != n { + t.Errorf("expected %d elements, got %d", n, count) + } + }) + + t.Run("Clone and Merge", func(t *testing.T) { + m := New[string, int]() + m.Set("a", 1) + m.Set("b", 2) + + // Test Clone + clone := m.Clone() + if val, ok := clone.Get("a"); !ok || val != 1 { + t.Error("Clone did not copy values correctly") + } + + // Test Merge + other := map[string]int{"c": 3, "d": 4} + m.Merge(other) + if val, ok := m.Get("c"); !ok || val != 3 { + t.Error("Merge did not add new values correctly") + } + }) + + t.Run("GetKeyWithValue", func(t *testing.T) { + m := New[string, int]() + m.Set("a", 1) + m.Set("b", 2) + + if key, ok := m.GetKeyWithValue(1); !ok || key != "a" { + t.Errorf("GetKeyWithValue(1) = (%v, %v), want (\"a\", true)", key, ok) + } + + if _, ok := m.GetKeyWithValue(3); ok { + t.Error("GetKeyWithValue(3) should return false") + } + }) + + t.Run("GetKeys", func(t *testing.T) { + m := New[string, int]() + m.Set("a", 1) + m.Set("b", 2) + m.Set("c", 3) + + values := m.GetKeys("a", "c", "missing") + if len(values) != 2 || values[0] != 1 || values[1] != 3 { + t.Errorf("GetKeys returned unexpected values: %v", values) + } + }) +} From 5b9371ccfd647bc535b4f9f8af9e0016a7b515eb Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 5 Feb 2025 04:30:38 +0700 Subject: [PATCH 02/18] test(maps): add bench test Signed-off-by: Dwi Siswanto --- maps/mapsutil_bench_test.go | 206 ++++++++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 maps/mapsutil_bench_test.go diff --git a/maps/mapsutil_bench_test.go b/maps/mapsutil_bench_test.go new file mode 100644 index 0000000..4722f7f --- /dev/null +++ b/maps/mapsutil_bench_test.go @@ -0,0 +1,206 @@ +package mapsutil + +import ( + "fmt" + "testing" +) + +var benchNumItems = []int{1000, 5000, 10_000, 100_000, 250_000, 500_000, 1_000_000} + +func BenchmarkGet(b *testing.B) { + for _, numItems := range benchNumItems { + b.Run(fmt.Sprintf("items=%d", numItems), func(b *testing.B) { + m := make(Map[string, int], numItems) + + // Pre-populate with test data + for i := 0; i < numItems; i++ { + m[fmt.Sprint(i)] = i + } + + b.ResetTimer() + b.Run("seq", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, _ = m.Get(fmt.Sprint(i % numItems)) + } + }) + + b.ResetTimer() + b.Run("parallel", func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + _, _ = m.Get(fmt.Sprint(i % numItems)) + i++ + } + }) + }) + }) + } +} + +func BenchmarkSyncMapGet(b *testing.B) { + for _, numItems := range benchNumItems { + b.Run(fmt.Sprintf("items=%d", numItems), func(b *testing.B) { + m := NewSyncLockMap[string, int]() + + // Pre-populate with test data + for i := 0; i < numItems; i++ { + _ = m.Set(fmt.Sprint(i), i) + } + + b.ResetTimer() + b.Run("seq", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, _ = m.Get(fmt.Sprint(i % numItems)) + } + }) + + b.ResetTimer() + b.Run("parallel", func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + _, _ = m.Get(fmt.Sprint(i % numItems)) + i++ + } + }) + }) + }) + } +} + +func BenchmarkSet(b *testing.B) { + b.Run("seq", func(b *testing.B) { + m := make(Map[string, int], b.N) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + m.Set(fmt.Sprint(i), i) + } + }) + + // no-op: non-concurrent map is not safe for concurrent writes + // b.Run("parallel", func(b *testing.B) { + // m := make(Map[string, int], b.N) + // b.ResetTimer() + + // b.RunParallel(func(pb *testing.PB) { + // i := 0 + // for pb.Next() { + // m.Set(fmt.Sprint(i), i) + // i++ + // } + // }) + // }) +} + +func BenchmarkSyncMapSet(b *testing.B) { + for _, numItems := range benchNumItems { + b.Run(fmt.Sprintf("items=%d", numItems), func(b *testing.B) { + b.Run("seq", func(b *testing.B) { + m := NewSyncLockMap[string, int]() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _ = m.Set(fmt.Sprint(i), i) + } + }) + + b.Run("parallel", func(b *testing.B) { + m := NewSyncLockMap[string, int]() + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + _ = m.Set(fmt.Sprint(i), i) + i++ + } + }) + }) + }) + } +} + +// func BenchmarkSyncMapConcurrent(b *testing.B) { +// for _, workers := range []int{2, 4, 8, 16, 32, 64} { +// b.Run(fmt.Sprintf("workers=%d", workers), func(b *testing.B) { +// m := NewSyncLockMap[string, int]() +// var wg sync.WaitGroup + +// b.ResetTimer() +// b.Run("Set", func(b *testing.B) { +// for i := 0; i < b.N; i++ { +// wg.Add(workers) +// for w := 0; w < workers; w++ { +// go func() { +// defer wg.Done() +// _ = m.Set("key", b.N) +// }() +// } +// wg.Wait() +// } +// }) + +// b.ResetTimer() +// b.Run("Get", func(b *testing.B) { +// for i := 0; i < b.N; i++ { +// wg.Add(workers) +// for w := 0; w < workers; w++ { +// go func() { +// defer wg.Done() +// _, _ = m.Get("key") +// }() +// } +// wg.Wait() +// } +// }) +// }) +// } +// } + +// func BenchmarkOps(b *testing.B) { +// items := 1_000_000 +// m := make(Map[string, int], items) + +// // Pre-populate +// for i := 0; i < items; i++ { +// m[fmt.Sprint(i)] = i +// } + +// b.ResetTimer() + +// b.Run("Has", func(b *testing.B) { +// b.RunParallel(func(pb *testing.PB) { +// i := 0 +// for pb.Next() { +// _ = m.Has(fmt.Sprint(i % items)) +// i++ +// } +// }) +// }) + +// b.Run("GetOrDefault", func(b *testing.B) { +// b.RunParallel(func(pb *testing.PB) { +// i := 0 +// for pb.Next() { +// _ = m.GetOrDefault(fmt.Sprint(i%items), -1) +// i++ +// } +// }) +// }) + +// b.Run("GetKeys", func(b *testing.B) { +// keys := make([]string, 100) +// for i := range keys { +// keys[i] = fmt.Sprint(i) +// } +// b.ResetTimer() + +// b.RunParallel(func(pb *testing.PB) { +// for pb.Next() { +// _ = m.GetKeys(keys...) +// } +// }) +// }) +// } From 99d911f51735f6d2c93d62c11d5a13f6fc6e0e58 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 5 Feb 2025 05:52:25 +0700 Subject: [PATCH 03/18] chore(swissmap): add BENCHMARK.md Signed-off-by: Dwi Siswanto --- swissmap/BENCHMARK.md | 135 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 swissmap/BENCHMARK.md diff --git a/swissmap/BENCHMARK.md b/swissmap/BENCHMARK.md new file mode 100644 index 0000000..654135a --- /dev/null +++ b/swissmap/BENCHMARK.md @@ -0,0 +1,135 @@ +# Benchmark + +* **swissmap** *(uses [cockroachdb/swiss](https://github.com/cockroachdb/swiss) under the hood)* + +``` +BenchmarkGet/items=1000/seq-16 8000854 139.2 ns/op 12 B/op 1 allocs/op +BenchmarkGet/items=1000/parallel-16 49186312 24.73 ns/op 12 B/op 1 allocs/op +BenchmarkGet/items=5000/seq-16 8156024 135.9 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=5000/parallel-16 45306098 26.74 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=10000/seq-16 8093172 140.4 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=10000/parallel-16 44395874 31.18 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=100000/seq-16 6251833 189.4 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=100000/parallel-16 39724492 31.15 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=250000/seq-16 5020261 222.7 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=250000/parallel-16 49155284 30.00 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=500000/seq-16 4082534 288.1 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=500000/parallel-16 42062390 29.94 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=1000000/seq-16 3348501 354.1 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=1000000/parallel-16 43152501 31.64 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=1000/WithThreadSafety/seq-16 8574296 139.3 ns/op 12 B/op 1 allocs/op +BenchmarkGet/items=1000/WithThreadSafety/parallel-16 10291254 138.4 ns/op 12 B/op 1 allocs/op +BenchmarkGet/items=5000/WithThreadSafety/seq-16 7859248 146.9 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=5000/WithThreadSafety/parallel-16 10370797 138.2 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=10000/WithThreadSafety/seq-16 7797219 151.9 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=10000/WithThreadSafety/parallel-16 9954980 143.8 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=100000/WithThreadSafety/seq-16 5443500 208.6 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=100000/WithThreadSafety/parallel-16 9368031 142.4 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=250000/WithThreadSafety/seq-16 4688386 277.2 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=250000/WithThreadSafety/parallel-16 9505594 138.6 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=500000/WithThreadSafety/seq-16 3644149 320.5 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=500000/WithThreadSafety/parallel-16 8204418 140.5 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=1000000/WithThreadSafety/seq-16 3098900 382.3 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=1000000/WithThreadSafety/parallel-16 8717754 138.7 ns/op 15 B/op 1 allocs/op +BenchmarkSet/items=1000/WithThreadSafety/seq-16 2570617 466.2 ns/op 37 B/op 2 allocs/op +BenchmarkSet/items=1000/WithThreadSafety/parallel-16 2163002 548.9 ns/op 15 B/op 1 allocs/op +BenchmarkSet/items=5000/WithThreadSafety/seq-16 2594163 465.3 ns/op 37 B/op 2 allocs/op +BenchmarkSet/items=5000/WithThreadSafety/parallel-16 2159650 529.8 ns/op 15 B/op 1 allocs/op +BenchmarkSet/items=10000/WithThreadSafety/seq-16 2610625 457.1 ns/op 36 B/op 2 allocs/op +BenchmarkSet/items=10000/WithThreadSafety/parallel-16 2265956 526.8 ns/op 15 B/op 1 allocs/op +BenchmarkSet/items=100000/WithThreadSafety/seq-16 2556045 482.4 ns/op 37 B/op 2 allocs/op +BenchmarkSet/items=100000/WithThreadSafety/parallel-16 2196990 541.3 ns/op 15 B/op 1 allocs/op +BenchmarkSet/items=250000/WithThreadSafety/seq-16 2680706 471.0 ns/op 36 B/op 2 allocs/op +BenchmarkSet/items=250000/WithThreadSafety/parallel-16 2444006 509.2 ns/op 15 B/op 1 allocs/op +BenchmarkSet/items=500000/WithThreadSafety/seq-16 2899224 452.2 ns/op 34 B/op 2 allocs/op +BenchmarkSet/items=500000/WithThreadSafety/parallel-16 2279362 525.6 ns/op 15 B/op 1 allocs/op +BenchmarkSet/items=1000000/WithThreadSafety/seq-16 3387404 463.5 ns/op 32 B/op 2 allocs/op +BenchmarkSet/items=1000000/WithThreadSafety/parallel-16 2167636 546.3 ns/op 15 B/op 1 allocs/op +``` + +* mapsutil (`4a4cbd9`) + +``` +BenchmarkGet/items=1000/seq-16 8214813 144.5 ns/op 12 B/op 1 allocs/op +BenchmarkGet/items=1000/parallel-16 52135828 24.30 ns/op 12 B/op 1 allocs/op +BenchmarkGet/items=5000/seq-16 7584444 157.0 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=5000/parallel-16 47813564 26.42 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=10000/seq-16 7216506 160.1 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=10000/parallel-16 46165209 28.49 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=100000/seq-16 6186141 195.1 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=100000/parallel-16 39932752 29.89 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=250000/seq-16 5442273 255.5 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=250000/parallel-16 51295606 28.32 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=500000/seq-16 4219646 263.2 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=500000/parallel-16 50865244 28.94 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=1000000/seq-16 4100514 298.2 ns/op 15 B/op 1 allocs/op +BenchmarkGet/items=1000000/parallel-16 49751544 30.14 ns/op 15 B/op 1 allocs/op +BenchmarkSyncMapGet/items=1000/seq-16 7747381 155.3 ns/op 12 B/op 1 allocs/op +BenchmarkSyncMapGet/items=1000/parallel-16 13463876 98.39 ns/op 12 B/op 1 allocs/op +BenchmarkSyncMapGet/items=5000/seq-16 6824139 174.3 ns/op 15 B/op 1 allocs/op +BenchmarkSyncMapGet/items=5000/parallel-16 10122182 120.5 ns/op 15 B/op 1 allocs/op +BenchmarkSyncMapGet/items=10000/seq-16 6453328 173.2 ns/op 15 B/op 1 allocs/op +BenchmarkSyncMapGet/items=10000/parallel-16 13732112 102.1 ns/op 15 B/op 1 allocs/op +BenchmarkSyncMapGet/items=100000/seq-16 5495906 212.5 ns/op 15 B/op 1 allocs/op +BenchmarkSyncMapGet/items=100000/parallel-16 11262956 119.9 ns/op 15 B/op 1 allocs/op +BenchmarkSyncMapGet/items=250000/seq-16 4041955 289.1 ns/op 15 B/op 1 allocs/op +BenchmarkSyncMapGet/items=250000/parallel-16 10759794 121.0 ns/op 15 B/op 1 allocs/op +BenchmarkSyncMapGet/items=500000/seq-16 3405672 325.4 ns/op 15 B/op 1 allocs/op +BenchmarkSyncMapGet/items=500000/parallel-16 13332009 97.60 ns/op 15 B/op 1 allocs/op +BenchmarkSyncMapGet/items=1000000/seq-16 3500380 346.9 ns/op 15 B/op 1 allocs/op +BenchmarkSyncMapGet/items=1000000/parallel-16 11079632 120.0 ns/op 15 B/op 1 allocs/op +BenchmarkSyncMapSet/items=1000/seq-16 1902699 646.8 ns/op 146 B/op 2 allocs/op +BenchmarkSyncMapSet/items=1000/parallel-16 2409667 497.8 ns/op 22 B/op 2 allocs/op +BenchmarkSyncMapSet/items=5000/seq-16 1929216 657.0 ns/op 144 B/op 2 allocs/op +BenchmarkSyncMapSet/items=5000/parallel-16 2402744 504.2 ns/op 22 B/op 2 allocs/op +BenchmarkSyncMapSet/items=10000/seq-16 2019657 638.3 ns/op 138 B/op 2 allocs/op +BenchmarkSyncMapSet/items=10000/parallel-16 2319279 527.0 ns/op 22 B/op 2 allocs/op +BenchmarkSyncMapSet/items=100000/seq-16 1825004 651.8 ns/op 151 B/op 2 allocs/op +BenchmarkSyncMapSet/items=100000/parallel-16 2501198 549.2 ns/op 22 B/op 2 allocs/op +BenchmarkSyncMapSet/items=250000/seq-16 1939453 640.4 ns/op 143 B/op 2 allocs/op +BenchmarkSyncMapSet/items=250000/parallel-16 2542819 546.9 ns/op 22 B/op 2 allocs/op +BenchmarkSyncMapSet/items=500000/seq-16 1949194 742.8 ns/op 143 B/op 2 allocs/op +BenchmarkSyncMapSet/items=500000/parallel-16 2523684 572.5 ns/op 22 B/op 2 allocs/op +BenchmarkSyncMapSet/items=1000000/seq-16 1987189 720.9 ns/op 140 B/op 2 allocs/op +BenchmarkSyncMapSet/items=1000000/parallel-16 2249832 507.2 ns/op 22 B/op 2 allocs/op +``` + +--- + +## key observations + +### perf + +* `Get` op + * `swissmap` is faster than `mapsutil` for small-to-medium datasets in sequence + * e.g., **139–151 ns/op** vs. **155–346 ns/op** at 1K–1M items + * `mapsutil` outperforms `swissmap` in parallel + * e.g., **98–120 ns/op** vs. **138–143 ns/op** for 1M items +* `Set` op + * `mapsutil` is slightly faster than `swissmap` in parallel but uses more memory + * `mapsutil` + * **497–572 ns/op** + * **22 B/op** + * `swissmap` + * **509–548 ns/op** + * **15 B/op** + * `swissmap` is significantly faster in sequence and *far more memory-efficient* + * `mapsutil` + * **497–572 ns/op** + * **138–151 B/op** + * `swissmap` + * **450–550 ns/op** + * **32–37 B/op** +* memory efficient + * `swissmap` uses **~15 B/op** for `Get` and **15–37 B/op** for `Set` *(lower in parallel)* + * `mapsutil` uses **12–15 B/op** for `Get` but **22–151 B/op** for `Set` (overhead in sequential `Set`) +* scalability + * `swissmap` maintains stable parallel `Get` perf regardless of item count (**~138–143 ns/op**) + +### conclusion + +1. use `mapsutil` if parallel `Get` perf is critical and memory overhead is *"acceptable"* +1. prefer `swissmap` for seq workloads, memory-sensitive apps *(especially `Set`)*, and consistent parallel `Get` perf across large datasets +1. `swissmap` trades slightly slower parallel `Get` for better memory efficiency in `Set`, while `mapsutil` prioritizes speed in concurrent r/w + +`mapsutil` for read-heavy, while `swissmap` for write-heavy scenarios or when memory efficiency is critical \ No newline at end of file From 57fc90d8cf1f4abf003a5088704281038cb2d483 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 5 Feb 2025 06:00:17 +0700 Subject: [PATCH 04/18] test(swissmap): missing `WithThreadSafety` opt in concurrent ops Signed-off-by: Dwi Siswanto --- swissmap/swissmap_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swissmap/swissmap_test.go b/swissmap/swissmap_test.go index 3ab37e2..12e3b51 100644 --- a/swissmap/swissmap_test.go +++ b/swissmap/swissmap_test.go @@ -40,7 +40,7 @@ func TestMap(t *testing.T) { }) t.Run("Concurrent operations", func(t *testing.T) { - m := New[int, int]() + m := New(WithThreadSafety[int, int]()) var wg sync.WaitGroup n := 1000 From 8b8642199f0c3a233f5404285ad28b572ab26f60 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 5 Feb 2025 08:50:06 +0700 Subject: [PATCH 05/18] feat(swissmap): implements JSON marshal/unmarshaller Signed-off-by: Dwi Siswanto --- go.mod | 6 +++ go.sum | 18 +++++++++ swissmap/locker.go | 29 ++++++++++++++ swissmap/options.go | 18 +++++++++ swissmap/swissmap.go | 79 ++++++++++++++++--------------------- swissmap/swissmap_test.go | 82 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 187 insertions(+), 45 deletions(-) create mode 100644 swissmap/locker.go create mode 100644 swissmap/options.go diff --git a/go.mod b/go.mod index 8c63baa..1e5c473 100644 --- a/go.mod +++ b/go.mod @@ -51,9 +51,12 @@ require ( github.com/andybalholm/brotli v1.0.6 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bits-and-blooms/bitset v1.13.0 // indirect + github.com/bytedance/sonic v1.12.8 // indirect + github.com/bytedance/sonic/loader v0.2.2 // indirect github.com/charmbracelet/lipgloss v0.13.0 // indirect github.com/charmbracelet/x/ansi v0.3.2 // indirect github.com/cloudflare/circl v1.3.7 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.11.4 // indirect @@ -63,6 +66,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect @@ -93,12 +97,14 @@ require ( github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ulikunitz/xz v0.5.11 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yuin/goldmark v1.7.4 // indirect github.com/yuin/goldmark-emoji v1.0.3 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.3.7 // indirect + golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index e92e4af..b868077 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,11 @@ github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJR github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY= github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs= +github.com/bytedance/sonic v1.12.8 h1:4xYRVRlXIgvSZ4e8iVTlMF5szgpXd4AfvuWgA8I8lgs= +github.com/bytedance/sonic v1.12.8/go.mod h1:uVvFidNmlt9+wa31S1urfwwthTWteBgG0hWuoKAXTx8= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.2 h1:jxAJuN9fOot/cyz5Q6dUuMJF5OqQ6+5GfA8FjjQ0R4o= +github.com/bytedance/sonic/loader v0.2.2/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs= @@ -55,6 +60,9 @@ github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObk github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 h1:Ew0znI2JatzKy52N1iS5muUsHkf2UJuhocH7uFW7jjs= github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -136,11 +144,15 @@ github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kljensen/snowball v0.8.0 h1:WU4cExxK6sNW33AiGdbn4e8RvloHrhkAssu2mVJ11kg= github.com/kljensen/snowball v0.8.0/go.mod h1:OGo5gFWjaeXqCu4iIrMl5OYip9XUJHGOU5eSkPjVg2A= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -257,6 +269,7 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= @@ -289,6 +302,8 @@ github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0h github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= @@ -325,6 +340,8 @@ go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -454,3 +471,4 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/swissmap/locker.go b/swissmap/locker.go new file mode 100644 index 0000000..8c12ba8 --- /dev/null +++ b/swissmap/locker.go @@ -0,0 +1,29 @@ +package swissmap + +// lock conditionally acquires the read lock if thread-safety is enabled +func (m *Map[K, V]) lock() { + if m.threadSafe { + m.mutex.Lock() + } +} + +// unlock conditionally releases the read lock if thread-safety is enabled +func (m *Map[K, V]) unlock() { + if m.threadSafe { + m.mutex.Unlock() + } +} + +// rLock conditionally acquires the read lock if thread-safety is enabled +func (m *Map[K, V]) rLock() { + if m.threadSafe { + m.mutex.RLock() + } +} + +// rUnlock conditionally releases the read lock if thread-safety is enabled +func (m *Map[K, V]) rUnlock() { + if m.threadSafe { + m.mutex.RUnlock() + } +} diff --git a/swissmap/options.go b/swissmap/options.go new file mode 100644 index 0000000..c88cada --- /dev/null +++ b/swissmap/options.go @@ -0,0 +1,18 @@ +package swissmap + +// Option represents a configuration option for the Map +type Option[K, V comparable] func(*Map[K, V]) + +// WithCapacity sets the initial capacity of the map +func WithCapacity[K, V comparable](capacity int) Option[K, V] { + return func(m *Map[K, V]) { + m.data.Init(capacity) + } +} + +// WithThreadSafety enables thread-safety for the map +func WithThreadSafety[K, V comparable]() Option[K, V] { + return func(m *Map[K, V]) { + m.threadSafe = true + } +} diff --git a/swissmap/swissmap.go b/swissmap/swissmap.go index 32063a8..ad3b16a 100644 --- a/swissmap/swissmap.go +++ b/swissmap/swissmap.go @@ -3,26 +3,10 @@ package swissmap import ( "sync" + "github.com/bytedance/sonic" "github.com/cockroachdb/swiss" ) -// Option represents a configuration option for the Map -type Option[K, V comparable] func(*Map[K, V]) - -// WithCapacity sets the initial capacity of the map -func WithCapacity[K, V comparable](capacity int) Option[K, V] { - return func(m *Map[K, V]) { - m.data.Init(capacity) - } -} - -// WithThreadSafety enables thread-safety for the map -func WithThreadSafety[K, V comparable]() Option[K, V] { - return func(m *Map[K, V]) { - m.threadSafe = true - } -} - // Map is a generic map implementation using swiss.Map with optional thread-safety type Map[K, V comparable] struct { mutex sync.RWMutex @@ -41,34 +25,6 @@ func New[K, V comparable](options ...Option[K, V]) *Map[K, V] { return m } -// lock conditionally acquires the read lock if thread-safety is enabled -func (m *Map[K, V]) lock() { - if m.threadSafe { - m.mutex.Lock() - } -} - -// unlock conditionally releases the read lock if thread-safety is enabled -func (m *Map[K, V]) unlock() { - if m.threadSafe { - m.mutex.Unlock() - } -} - -// rLock conditionally acquires the read lock if thread-safety is enabled -func (m *Map[K, V]) rLock() { - if m.threadSafe { - m.mutex.RLock() - } -} - -// rUnlock conditionally releases the read lock if thread-safety is enabled -func (m *Map[K, V]) rUnlock() { - if m.threadSafe { - m.mutex.RUnlock() - } -} - // Clear removes all elements from the map func (m *Map[K, V]) Clear() bool { m.lock() @@ -76,6 +32,7 @@ func (m *Map[K, V]) Clear() bool { hadElements := m.data.Len() > 0 m.data.Clear() + return hadElements } @@ -186,3 +143,35 @@ func (m *Map[K, V]) Set(key K, value V) { m.data.Put(key, value) } + +// MarshalJSON marshals the map to JSON +func (m *Map[K, V]) MarshalJSON() ([]byte, error) { + m.rLock() + defer m.rUnlock() + + target := make(map[K]V) + + m.data.All(func(key K, value V) bool { + target[key] = value + + return true + }) + + return sonic.Marshal(target) +} + +// UnmarshalJSON unmarshals the map from JSON +func (m *Map[K, V]) UnmarshalJSON(buf []byte) error { + m.lock() + defer m.unlock() + + target := make(map[K]V) + + if err := sonic.Unmarshal(buf, &target); err != nil { + return err + } + + m.Merge(target) + + return nil +} diff --git a/swissmap/swissmap_test.go b/swissmap/swissmap_test.go index 12e3b51..4ecb622 100644 --- a/swissmap/swissmap_test.go +++ b/swissmap/swissmap_test.go @@ -122,3 +122,85 @@ func TestMap(t *testing.T) { } }) } + +func TestMapJSON(t *testing.T) { + t.Run("Marshal and Unmarshal", func(t *testing.T) { + m := New[string, int]() + m.Set("one", 1) + m.Set("two", 2) + m.Set("three", 3) + + // Test marshaling + data, err := m.MarshalJSON() + if err != nil { + t.Fatalf("MarshalJSON failed: %v", err) + } + t.Logf("marshaled data: %s", data) + + // Test unmarshaling into new map + newMap := New[string, int]() + err = newMap.UnmarshalJSON(data) + if err != nil { + t.Fatalf("UnmarshalJSON failed: %v", err) + } + + data2, err := newMap.MarshalJSON() + if err != nil { + t.Fatalf("MarshalJSON failed for unmarshaled map: %v", err) + } + + t.Logf("unmarshaled data: %s", data2) + + // Verify contents + expected := map[string]int{"one": 1, "two": 2, "three": 3} + for k, v := range expected { + if val, ok := newMap.Get(k); !ok || val != v { + t.Errorf("expected %s=%d, got %d (exists: %v)", k, v, val, ok) + } + } + }) + + t.Run("Empty map", func(t *testing.T) { + m := New[string, string]() + + data, err := m.MarshalJSON() + if err != nil { + t.Fatalf("MarshalJSON failed for empty map: %v", err) + } + + newMap := New[string, string]() + err = newMap.UnmarshalJSON(data) + if err != nil { + t.Fatalf("UnmarshalJSON failed for empty map: %v", err) + } + + if !newMap.IsEmpty() { + t.Error("unmarshaled map should be empty") + } + }) + + t.Run("Complex types", func(t *testing.T) { + type Complex struct { + ID int + Name string + } + m := New[string, Complex]() + m.Set("item1", Complex{ID: 1, Name: "test1"}) + m.Set("item2", Complex{ID: 2, Name: "test2"}) + + data, err := m.MarshalJSON() + if err != nil { + t.Fatalf("MarshalJSON failed for complex types: %v", err) + } + + newMap := New[string, Complex]() + err = newMap.UnmarshalJSON(data) + if err != nil { + t.Fatalf("UnmarshalJSON failed for complex types: %v", err) + } + + if val, ok := newMap.Get("item1"); !ok || val.ID != 1 || val.Name != "test1" { + t.Error("complex type was not correctly unmarshaled") + } + }) +} From 72ef9f3558febd63439fc8b2c4edb106d8306d75 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 5 Feb 2025 10:28:44 +0700 Subject: [PATCH 06/18] feat(swissmap): add `WithSortMapKeys` option Signed-off-by: Dwi Siswanto --- swissmap/options.go | 11 ++++++++++ swissmap/swissmap.go | 33 +++++++++++++++++++++--------- swissmap/swissmap_test.go | 42 ++++++++++++++++++++++++++++++++++++++- swissmap/utils.go | 18 +++++++++++++++++ 4 files changed, 94 insertions(+), 10 deletions(-) create mode 100644 swissmap/utils.go diff --git a/swissmap/options.go b/swissmap/options.go index c88cada..2e2e76b 100644 --- a/swissmap/options.go +++ b/swissmap/options.go @@ -16,3 +16,14 @@ func WithThreadSafety[K, V comparable]() Option[K, V] { m.threadSafe = true } } + +// WithSortMapKeys enables sorting of map keys +func WithSortMapKeys[K, V comparable]() Option[K, V] { + cfg := getDefaultSonicConfig() + cfg.SortMapKeys = true + + return func(m *Map[K, V]) { + m.sorted = true + m.api = cfg.Froze() + } +} diff --git a/swissmap/swissmap.go b/swissmap/swissmap.go index ad3b16a..29efe18 100644 --- a/swissmap/swissmap.go +++ b/swissmap/swissmap.go @@ -7,16 +7,21 @@ import ( "github.com/cockroachdb/swiss" ) -// Map is a generic map implementation using swiss.Map with optional thread-safety +// Map is a generic map implementation using swiss.Map with additional [Option]s type Map[K, V comparable] struct { + api sonic.API + data *swiss.Map[K, V] mutex sync.RWMutex threadSafe bool - data *swiss.Map[K, V] + sorted bool } // New creates a new Map with the given options func New[K, V comparable](options ...Option[K, V]) *Map[K, V] { - m := &Map[K, V]{data: swiss.New[K, V](0)} + m := &Map[K, V]{ + data: swiss.New[K, V](0), + api: getDefaultSonicConfig().Froze(), + } for _, opt := range options { opt(m) @@ -108,6 +113,17 @@ func (m *Map[K, V]) GetOrDefault(key K, defaultValue V) V { return defaultValue } +// GetByIndex retrieves a value by its index +// +// The index is 0-based and must be less than the number of elements in the map +func (m *Map[K, V]) GetByIndex(idx int) (V, bool) { + // TODO(dwisiswant0): Implement this method + + var value V + + return value, false +} + // Has checks if a key exists in the map func (m *Map[K, V]) Has(key K) bool { m.rLock() @@ -128,11 +144,8 @@ func (m *Map[K, V]) IsEmpty() bool { // Merge adds all key/value pairs from the input map func (m *Map[K, V]) Merge(n map[K]V) { - m.lock() - defer m.unlock() - for k, v := range n { - m.data.Put(k, v) + m.Set(k, v) } } @@ -157,17 +170,19 @@ func (m *Map[K, V]) MarshalJSON() ([]byte, error) { return true }) - return sonic.Marshal(target) + return m.api.Marshal(target) } // UnmarshalJSON unmarshals the map from JSON +// +// The map is merged with the input data. func (m *Map[K, V]) UnmarshalJSON(buf []byte) error { m.lock() defer m.unlock() target := make(map[K]V) - if err := sonic.Unmarshal(buf, &target); err != nil { + if err := m.api.Unmarshal(buf, &target); err != nil { return err } diff --git a/swissmap/swissmap_test.go b/swissmap/swissmap_test.go index 4ecb622..1885b94 100644 --- a/swissmap/swissmap_test.go +++ b/swissmap/swissmap_test.go @@ -160,6 +160,46 @@ func TestMapJSON(t *testing.T) { } }) + t.Run("WithSortMapKeys", func(t *testing.T) { + t.SkipNow() + + m := New(WithSortMapKeys[string, int]()) + + // Insert items in random order + m.Set("zebra", 1) + m.Set("alpha", 2) + m.Set("beta", 3) + + data, err := m.MarshalJSON() + if err != nil { + t.Fatalf("MarshalJSON failed with sorted keys: %v", err) + } + t.Logf("marshaled data with sorted keys: %s", data) + + // Test getting by index with sorted keys + if v, ok := m.GetByIndex(0); !ok || v != 2 { + t.Errorf("GetByIndex(0) = (%v, %v), want (2, true)", v, ok) + } + + if v, ok := m.GetByIndex(1); !ok || v != 3 { + t.Errorf("GetByIndex(1) = (%v, %v), want (3, true)", v, ok) + } + + if v, ok := m.GetByIndex(2); !ok || v != 1 { + t.Errorf("GetByIndex(2) = (%v, %v), want (1, true)", v, ok) + } + + // Test out of bounds index + if _, ok := m.GetByIndex(3); ok { + t.Error("GetByIndex(3) should return false for out of bounds index") + } + + // Test negative index + if _, ok := m.GetByIndex(-1); ok { + t.Error("GetByIndex(-1) should return false for negative index") + } + }) + t.Run("Empty map", func(t *testing.T) { m := New[string, string]() @@ -181,8 +221,8 @@ func TestMapJSON(t *testing.T) { t.Run("Complex types", func(t *testing.T) { type Complex struct { - ID int Name string + ID int } m := New[string, Complex]() m.Set("item1", Complex{ID: 1, Name: "test1"}) diff --git a/swissmap/utils.go b/swissmap/utils.go new file mode 100644 index 0000000..12bdc29 --- /dev/null +++ b/swissmap/utils.go @@ -0,0 +1,18 @@ +package swissmap + +import "github.com/bytedance/sonic" + +// getDefaultSonicConfig provides the default configuration for the Sonic +// library. +// +// This function returns a [sonic.Config] instance with standard `encoding/json` +// settings but with unsorted map keys. You may want to use the [WithSortMapKeys] +// option to enable sorting of map keys. +func getDefaultSonicConfig() sonic.Config { + return sonic.Config{ + EscapeHTML: true, + CompactMarshaler: true, + CopyString: true, + ValidateString: true, + } +} From de970a24ea3337f6ca5c15fcb60f9ba612ae8317 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 5 Feb 2025 20:32:45 +0700 Subject: [PATCH 07/18] feat(swissmap): update `Map` type constraints to `ComparableOrdered` & `any` Signed-off-by: Dwi Siswanto --- swissmap/options.go | 8 ++++---- swissmap/swissmap.go | 15 ++++++++++++--- swissmap/swissmap_bench_test.go | 2 +- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/swissmap/options.go b/swissmap/options.go index 2e2e76b..23077ed 100644 --- a/swissmap/options.go +++ b/swissmap/options.go @@ -1,24 +1,24 @@ package swissmap // Option represents a configuration option for the Map -type Option[K, V comparable] func(*Map[K, V]) +type Option[K ComparableOrdered, V any] func(*Map[K, V]) // WithCapacity sets the initial capacity of the map -func WithCapacity[K, V comparable](capacity int) Option[K, V] { +func WithCapacity[K ComparableOrdered, V any](capacity int) Option[K, V] { return func(m *Map[K, V]) { m.data.Init(capacity) } } // WithThreadSafety enables thread-safety for the map -func WithThreadSafety[K, V comparable]() Option[K, V] { +func WithThreadSafety[K ComparableOrdered, V any]() Option[K, V] { return func(m *Map[K, V]) { m.threadSafe = true } } // WithSortMapKeys enables sorting of map keys -func WithSortMapKeys[K, V comparable]() Option[K, V] { +func WithSortMapKeys[K ComparableOrdered, V any]() Option[K, V] { cfg := getDefaultSonicConfig() cfg.SortMapKeys = true diff --git a/swissmap/swissmap.go b/swissmap/swissmap.go index 29efe18..856cf73 100644 --- a/swissmap/swissmap.go +++ b/swissmap/swissmap.go @@ -1,14 +1,23 @@ package swissmap import ( + "cmp" + "reflect" "sync" "github.com/bytedance/sonic" "github.com/cockroachdb/swiss" ) +// ComparableOrdered is an interface that extends [cmp.Ordered] with the +// comparable interface +type ComparableOrdered interface { + cmp.Ordered + comparable +} + // Map is a generic map implementation using swiss.Map with additional [Option]s -type Map[K, V comparable] struct { +type Map[K ComparableOrdered, V any] struct { api sonic.API data *swiss.Map[K, V] mutex sync.RWMutex @@ -17,7 +26,7 @@ type Map[K, V comparable] struct { } // New creates a new Map with the given options -func New[K, V comparable](options ...Option[K, V]) *Map[K, V] { +func New[K ComparableOrdered, V any](options ...Option[K, V]) *Map[K, V] { m := &Map[K, V]{ data: swiss.New[K, V](0), api: getDefaultSonicConfig().Froze(), @@ -73,7 +82,7 @@ func (m *Map[K, V]) GetKeyWithValue(value V) (K, bool) { var found bool m.data.All(func(key K, v V) bool { - if v == value { + if reflect.DeepEqual(v, value) { foundKey = key found = true diff --git a/swissmap/swissmap_bench_test.go b/swissmap/swissmap_bench_test.go index b6d19ee..ff1657b 100644 --- a/swissmap/swissmap_bench_test.go +++ b/swissmap/swissmap_bench_test.go @@ -7,7 +7,7 @@ import ( var benchNumItems = []int{1000, 5000, 10_000, 100_000, 250_000, 500_000, 1_000_000} -func createMaps[K, V comparable](numItems int, threadSafe bool) *Map[K, V] { +func createMaps[K ComparableOrdered, V any](numItems int, threadSafe bool) *Map[K, V] { options := []Option[K, V]{ WithCapacity[K, V](numItems), } From 1ced3161a518dcb8a2c52c8d95327e62fc1f437f Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Wed, 5 Feb 2025 20:52:15 +0700 Subject: [PATCH 08/18] feat(swissmap): add key tracking & `GetByIndex` method Signed-off-by: Dwi Siswanto --- swissmap/swissmap.go | 32 ++++++++++++++++++++++++++++++-- swissmap/swissmap_test.go | 2 +- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/swissmap/swissmap.go b/swissmap/swissmap.go index 856cf73..67edc42 100644 --- a/swissmap/swissmap.go +++ b/swissmap/swissmap.go @@ -3,6 +3,7 @@ package swissmap import ( "cmp" "reflect" + "slices" "sync" "github.com/bytedance/sonic" @@ -23,6 +24,7 @@ type Map[K ComparableOrdered, V any] struct { mutex sync.RWMutex threadSafe bool sorted bool + keys []K } // New creates a new Map with the given options @@ -36,6 +38,14 @@ func New[K ComparableOrdered, V any](options ...Option[K, V]) *Map[K, V] { opt(m) } + // TODO(dwisiswant0): Add check for comparable key type + // if m.sorted { + // var k K + // if !reflect.TypeOf(k).Comparable() { + // panic("key type must be comparable for sorted map") + // } + // } + return m } @@ -45,7 +55,9 @@ func (m *Map[K, V]) Clear() bool { defer m.unlock() hadElements := m.data.Len() > 0 + m.data.Clear() + m.keys = []K{} return hadElements } @@ -126,11 +138,17 @@ func (m *Map[K, V]) GetOrDefault(key K, defaultValue V) V { // // The index is 0-based and must be less than the number of elements in the map func (m *Map[K, V]) GetByIndex(idx int) (V, bool) { - // TODO(dwisiswant0): Implement this method + m.rLock() + defer m.rUnlock() var value V - return value, false + // Return early if index out of range + if idx < 0 || idx >= m.data.Len() { + return value, false + } + + return m.data.Get(m.keys[idx]) } // Has checks if a key exists in the map @@ -163,6 +181,16 @@ func (m *Map[K, V]) Set(key K, value V) { m.lock() defer m.unlock() + if !m.Has(key) { + m.keys = append(m.keys, key) + if m.sorted { + // NOTE(dwisiswant0): It may cause a panic if the key is not comparable + slices.SortStableFunc(m.keys, func(a, b K) int { + return cmp.Compare(a, b) + }) + } + } + m.data.Put(key, value) } diff --git a/swissmap/swissmap_test.go b/swissmap/swissmap_test.go index 1885b94..53dfb44 100644 --- a/swissmap/swissmap_test.go +++ b/swissmap/swissmap_test.go @@ -161,7 +161,7 @@ func TestMapJSON(t *testing.T) { }) t.Run("WithSortMapKeys", func(t *testing.T) { - t.SkipNow() + // t.SkipNow() m := New(WithSortMapKeys[string, int]()) From 5cf499034b52b15688f8bd6947741146e0ce5b29 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 6 Feb 2025 02:14:46 +0700 Subject: [PATCH 09/18] fix(swissmap): `Map.Set` deadlock Signed-off-by: Dwi Siswanto --- swissmap/swissmap.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/swissmap/swissmap.go b/swissmap/swissmap.go index 67edc42..f8a1f67 100644 --- a/swissmap/swissmap.go +++ b/swissmap/swissmap.go @@ -178,10 +178,12 @@ func (m *Map[K, V]) Merge(n map[K]V) { // Set inserts or updates a key/value pair func (m *Map[K, V]) Set(key K, value V) { + exists := m.Has(key) + m.lock() defer m.unlock() - if !m.Has(key) { + if !exists { m.keys = append(m.keys, key) if m.sorted { // NOTE(dwisiswant0): It may cause a panic if the key is not comparable From 2f05f0c7f7b9cfd66f7d454fafae815de57f8ece Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 6 Feb 2025 02:28:49 +0700 Subject: [PATCH 10/18] chore(swissmap): autofix field alignment Signed-off-by: Dwi Siswanto --- swissmap/swissmap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swissmap/swissmap.go b/swissmap/swissmap.go index f8a1f67..aed5cb6 100644 --- a/swissmap/swissmap.go +++ b/swissmap/swissmap.go @@ -21,10 +21,10 @@ type ComparableOrdered interface { type Map[K ComparableOrdered, V any] struct { api sonic.API data *swiss.Map[K, V] + keys []K mutex sync.RWMutex threadSafe bool sorted bool - keys []K } // New creates a new Map with the given options From dfee10189edc244f62a1d7006d6d25e489ca5559 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 6 Feb 2025 02:32:19 +0700 Subject: [PATCH 11/18] perf(swissmap): reuse existing slice cap Signed-off-by: Dwi Siswanto --- swissmap/swissmap.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/swissmap/swissmap.go b/swissmap/swissmap.go index aed5cb6..5767f70 100644 --- a/swissmap/swissmap.go +++ b/swissmap/swissmap.go @@ -55,9 +55,10 @@ func (m *Map[K, V]) Clear() bool { defer m.unlock() hadElements := m.data.Len() > 0 - m.data.Clear() - m.keys = []K{} + + // Reuse existing slice capacity + m.keys = m.keys[:0] return hadElements } From ac1f5df5885d99c8990c26adb6acdea1852d5ae7 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 6 Feb 2025 02:40:35 +0700 Subject: [PATCH 12/18] feat(swissmap): deferred unlocks Signed-off-by: Dwi Siswanto --- swissmap/locker.go | 14 +++++++++-- swissmap/swissmap.go | 60 ++++++++++++++++++++++++++------------------ 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/swissmap/locker.go b/swissmap/locker.go index 8c12ba8..aceda0e 100644 --- a/swissmap/locker.go +++ b/swissmap/locker.go @@ -1,10 +1,15 @@ package swissmap // lock conditionally acquires the read lock if thread-safety is enabled -func (m *Map[K, V]) lock() { +func (m *Map[K, V]) lock() bool { + var locked bool + if m.threadSafe { m.mutex.Lock() + locked = true } + + return locked } // unlock conditionally releases the read lock if thread-safety is enabled @@ -15,10 +20,15 @@ func (m *Map[K, V]) unlock() { } // rLock conditionally acquires the read lock if thread-safety is enabled -func (m *Map[K, V]) rLock() { +func (m *Map[K, V]) rLock() bool { + var locked bool + if m.threadSafe { m.mutex.RLock() + locked = true } + + return locked } // rUnlock conditionally releases the read lock if thread-safety is enabled diff --git a/swissmap/swissmap.go b/swissmap/swissmap.go index 5767f70..ff7b6b2 100644 --- a/swissmap/swissmap.go +++ b/swissmap/swissmap.go @@ -51,8 +51,9 @@ func New[K ComparableOrdered, V any](options ...Option[K, V]) *Map[K, V] { // Clear removes all elements from the map func (m *Map[K, V]) Clear() bool { - m.lock() - defer m.unlock() + if m.lock() { + defer m.unlock() + } hadElements := m.data.Len() > 0 m.data.Clear() @@ -65,8 +66,9 @@ func (m *Map[K, V]) Clear() bool { // Clone returns a new Map with a copy of the underlying data func (m *Map[K, V]) Clone() *Map[K, V] { - m.rLock() - defer m.rUnlock() + if m.rLock() { + defer m.rUnlock() + } clone := New[K, V]() m.data.All(func(key K, value V) bool { @@ -80,16 +82,18 @@ func (m *Map[K, V]) Clone() *Map[K, V] { // Get retrieves a value from the map func (m *Map[K, V]) Get(key K) (V, bool) { - m.rLock() - defer m.rUnlock() + if m.rLock() { + defer m.rUnlock() + } return m.data.Get(key) } // GetKeyWithValue retrieves the first key associated with the given value func (m *Map[K, V]) GetKeyWithValue(value V) (K, bool) { - m.rLock() - defer m.rUnlock() + if m.rLock() { + defer m.rUnlock() + } var foundKey K var found bool @@ -110,8 +114,9 @@ func (m *Map[K, V]) GetKeyWithValue(value V) (K, bool) { // GetKeys returns values for the given keys func (m *Map[K, V]) GetKeys(keys ...K) []V { - m.rLock() - defer m.rUnlock() + if m.rLock() { + defer m.rUnlock() + } result := make([]V, 0, len(keys)) for _, key := range keys { @@ -125,8 +130,9 @@ func (m *Map[K, V]) GetKeys(keys ...K) []V { // GetOrDefault returns the value for key or defaultValue if key is not found func (m *Map[K, V]) GetOrDefault(key K, defaultValue V) V { - m.rLock() - defer m.rUnlock() + if m.rLock() { + defer m.rUnlock() + } if val, ok := m.data.Get(key); ok { return val @@ -139,8 +145,9 @@ func (m *Map[K, V]) GetOrDefault(key K, defaultValue V) V { // // The index is 0-based and must be less than the number of elements in the map func (m *Map[K, V]) GetByIndex(idx int) (V, bool) { - m.rLock() - defer m.rUnlock() + if m.rLock() { + defer m.rUnlock() + } var value V @@ -154,8 +161,9 @@ func (m *Map[K, V]) GetByIndex(idx int) (V, bool) { // Has checks if a key exists in the map func (m *Map[K, V]) Has(key K) bool { - m.rLock() - defer m.rUnlock() + if m.rLock() { + defer m.rUnlock() + } _, ok := m.data.Get(key) @@ -164,8 +172,9 @@ func (m *Map[K, V]) Has(key K) bool { // IsEmpty returns true if the map contains no elements func (m *Map[K, V]) IsEmpty() bool { - m.rLock() - defer m.rUnlock() + if m.rLock() { + defer m.rUnlock() + } return m.data.Len() == 0 } @@ -181,8 +190,9 @@ func (m *Map[K, V]) Merge(n map[K]V) { func (m *Map[K, V]) Set(key K, value V) { exists := m.Has(key) - m.lock() - defer m.unlock() + if m.lock() { + defer m.unlock() + } if !exists { m.keys = append(m.keys, key) @@ -199,8 +209,9 @@ func (m *Map[K, V]) Set(key K, value V) { // MarshalJSON marshals the map to JSON func (m *Map[K, V]) MarshalJSON() ([]byte, error) { - m.rLock() - defer m.rUnlock() + if m.rLock() { + defer m.rUnlock() + } target := make(map[K]V) @@ -217,8 +228,9 @@ func (m *Map[K, V]) MarshalJSON() ([]byte, error) { // // The map is merged with the input data. func (m *Map[K, V]) UnmarshalJSON(buf []byte) error { - m.lock() - defer m.unlock() + if m.lock() { + defer m.unlock() + } target := make(map[K]V) From 4a3fb62ff1af0aeabda30723db2edcd2506c447a Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 6 Feb 2025 05:01:14 +0700 Subject: [PATCH 13/18] feat(swissmap): prealloc `Map.keys` within `WithCapacity` Signed-off-by: Dwi Siswanto --- swissmap/options.go | 1 + 1 file changed, 1 insertion(+) diff --git a/swissmap/options.go b/swissmap/options.go index 23077ed..5b6eef5 100644 --- a/swissmap/options.go +++ b/swissmap/options.go @@ -7,6 +7,7 @@ type Option[K ComparableOrdered, V any] func(*Map[K, V]) func WithCapacity[K ComparableOrdered, V any](capacity int) Option[K, V] { return func(m *Map[K, V]) { m.data.Init(capacity) + m.keys = make([]K, 0, capacity) } } From 600cf9a2b51bbe6218ad5ed90d9b2ee924adcfb4 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Thu, 6 Feb 2025 05:17:32 +0700 Subject: [PATCH 14/18] chore(swissmap): `WithThreadSafety` -> `WithConcurrentAccess` and `m.threadSafe` -> `m.concurrent` Signed-off-by: Dwi Siswanto --- swissmap/locker.go | 8 ++++---- swissmap/options.go | 6 +++--- swissmap/swissmap.go | 2 +- swissmap/swissmap_bench_test.go | 4 ++-- swissmap/swissmap_test.go | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/swissmap/locker.go b/swissmap/locker.go index aceda0e..bedae10 100644 --- a/swissmap/locker.go +++ b/swissmap/locker.go @@ -4,7 +4,7 @@ package swissmap func (m *Map[K, V]) lock() bool { var locked bool - if m.threadSafe { + if m.concurrent { m.mutex.Lock() locked = true } @@ -14,7 +14,7 @@ func (m *Map[K, V]) lock() bool { // unlock conditionally releases the read lock if thread-safety is enabled func (m *Map[K, V]) unlock() { - if m.threadSafe { + if m.concurrent { m.mutex.Unlock() } } @@ -23,7 +23,7 @@ func (m *Map[K, V]) unlock() { func (m *Map[K, V]) rLock() bool { var locked bool - if m.threadSafe { + if m.concurrent { m.mutex.RLock() locked = true } @@ -33,7 +33,7 @@ func (m *Map[K, V]) rLock() bool { // rUnlock conditionally releases the read lock if thread-safety is enabled func (m *Map[K, V]) rUnlock() { - if m.threadSafe { + if m.concurrent { m.mutex.RUnlock() } } diff --git a/swissmap/options.go b/swissmap/options.go index 5b6eef5..1c4884b 100644 --- a/swissmap/options.go +++ b/swissmap/options.go @@ -11,10 +11,10 @@ func WithCapacity[K ComparableOrdered, V any](capacity int) Option[K, V] { } } -// WithThreadSafety enables thread-safety for the map -func WithThreadSafety[K ComparableOrdered, V any]() Option[K, V] { +// WithConcurrentAccess enables safe concurrent access to the [Map] +func WithConcurrentAccess[K ComparableOrdered, V any]() Option[K, V] { return func(m *Map[K, V]) { - m.threadSafe = true + m.concurrent = true } } diff --git a/swissmap/swissmap.go b/swissmap/swissmap.go index ff7b6b2..420c84c 100644 --- a/swissmap/swissmap.go +++ b/swissmap/swissmap.go @@ -23,7 +23,7 @@ type Map[K ComparableOrdered, V any] struct { data *swiss.Map[K, V] keys []K mutex sync.RWMutex - threadSafe bool + concurrent bool sorted bool } diff --git a/swissmap/swissmap_bench_test.go b/swissmap/swissmap_bench_test.go index ff1657b..9a8588b 100644 --- a/swissmap/swissmap_bench_test.go +++ b/swissmap/swissmap_bench_test.go @@ -13,7 +13,7 @@ func createMaps[K ComparableOrdered, V any](numItems int, threadSafe bool) *Map[ } if threadSafe { - options = append(options, WithThreadSafety[K, V]()) + options = append(options, WithConcurrentAccess[K, V]()) } return New[K, V](options...) @@ -45,7 +45,7 @@ func BenchmarkGet(b *testing.B) { }) }) - b.Run("WithThreadSafety", func(b *testing.B) { + b.Run("WithConcurrentAccess", func(b *testing.B) { m := createMaps[string, int](numItems, true) for i := 0; i < numItems; i++ { m.Set(fmt.Sprint(i), i) diff --git a/swissmap/swissmap_test.go b/swissmap/swissmap_test.go index 53dfb44..babb619 100644 --- a/swissmap/swissmap_test.go +++ b/swissmap/swissmap_test.go @@ -40,7 +40,7 @@ func TestMap(t *testing.T) { }) t.Run("Concurrent operations", func(t *testing.T) { - m := New(WithThreadSafety[int, int]()) + m := New(WithConcurrentAccess[int, int]()) var wg sync.WaitGroup n := 1000 From b190f599734f398fb8a2e13ccf3ca35fef9de78d Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sat, 8 Feb 2025 02:09:24 +0700 Subject: [PATCH 15/18] feat(swissmap): get `Map.keys` conditionally and prealloc those Signed-off-by: Dwi Siswanto --- swissmap/swissmap.go | 94 ++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/swissmap/swissmap.go b/swissmap/swissmap.go index 420c84c..b0827ea 100644 --- a/swissmap/swissmap.go +++ b/swissmap/swissmap.go @@ -22,7 +22,7 @@ type Map[K ComparableOrdered, V any] struct { api sonic.API data *swiss.Map[K, V] keys []K - mutex sync.RWMutex + mutex sync.Mutex concurrent bool sorted bool } @@ -59,17 +59,15 @@ func (m *Map[K, V]) Clear() bool { m.data.Clear() // Reuse existing slice capacity - m.keys = m.keys[:0] + if m.sorted { + m.keys = m.keys[:0] + } return hadElements } // Clone returns a new Map with a copy of the underlying data func (m *Map[K, V]) Clone() *Map[K, V] { - if m.rLock() { - defer m.rUnlock() - } - clone := New[K, V]() m.data.All(func(key K, value V) bool { clone.data.Put(key, value) @@ -82,19 +80,11 @@ func (m *Map[K, V]) Clone() *Map[K, V] { // Get retrieves a value from the map func (m *Map[K, V]) Get(key K) (V, bool) { - if m.rLock() { - defer m.rUnlock() - } - return m.data.Get(key) } // GetKeyWithValue retrieves the first key associated with the given value func (m *Map[K, V]) GetKeyWithValue(value V) (K, bool) { - if m.rLock() { - defer m.rUnlock() - } - var foundKey K var found bool @@ -114,10 +104,6 @@ func (m *Map[K, V]) GetKeyWithValue(value V) (K, bool) { // GetKeys returns values for the given keys func (m *Map[K, V]) GetKeys(keys ...K) []V { - if m.rLock() { - defer m.rUnlock() - } - result := make([]V, 0, len(keys)) for _, key := range keys { if val, ok := m.data.Get(key); ok { @@ -130,10 +116,6 @@ func (m *Map[K, V]) GetKeys(keys ...K) []V { // GetOrDefault returns the value for key or defaultValue if key is not found func (m *Map[K, V]) GetOrDefault(key K, defaultValue V) V { - if m.rLock() { - defer m.rUnlock() - } - if val, ok := m.data.Get(key); ok { return val } @@ -145,26 +127,38 @@ func (m *Map[K, V]) GetOrDefault(key K, defaultValue V) V { // // The index is 0-based and must be less than the number of elements in the map func (m *Map[K, V]) GetByIndex(idx int) (V, bool) { - if m.rLock() { - defer m.rUnlock() - } - var value V + var ok bool = false // Return early if index out of range if idx < 0 || idx >= m.data.Len() { - return value, false + return value, ok + } + + if m.sorted { + value, _ = m.data.Get(m.keys[idx]) + ok = true + } else { + i := 0 + m.data.All(func(key K, val V) bool { + if i == idx { + value = val + return false + } + + i++ + + return true + }) + + ok = (i == idx) } - return m.data.Get(m.keys[idx]) + return value, ok } // Has checks if a key exists in the map func (m *Map[K, V]) Has(key K) bool { - if m.rLock() { - defer m.rUnlock() - } - _, ok := m.data.Get(key) return ok @@ -172,10 +166,6 @@ func (m *Map[K, V]) Has(key K) bool { // IsEmpty returns true if the map contains no elements func (m *Map[K, V]) IsEmpty() bool { - if m.rLock() { - defer m.rUnlock() - } - return m.data.Len() == 0 } @@ -188,32 +178,42 @@ func (m *Map[K, V]) Merge(n map[K]V) { // Set inserts or updates a key/value pair func (m *Map[K, V]) Set(key K, value V) { - exists := m.Has(key) - if m.lock() { defer m.unlock() } - if !exists { - m.keys = append(m.keys, key) - if m.sorted { + m.data.Put(key, value) + + if m.sorted { + if exists := m.Has(key); !exists { + m.keys = append(m.keys, key) // NOTE(dwisiswant0): It may cause a panic if the key is not comparable slices.SortStableFunc(m.keys, func(a, b K) int { return cmp.Compare(a, b) }) } } +} - m.data.Put(key, value) +// Iterate iterates over the [Map] +func (m *Map[K, V]) Iterate(fn func(key K, value V) bool) { + if m.sorted { + for _, key := range m.keys { + value, ok := m.data.Get(key) + if ok && !fn(key, value) { + break + } + } + } else { + m.data.All(func(key K, value V) bool { + return fn(key, value) + }) + } } // MarshalJSON marshals the map to JSON func (m *Map[K, V]) MarshalJSON() ([]byte, error) { - if m.rLock() { - defer m.rUnlock() - } - - target := make(map[K]V) + target := make(map[K]V, m.data.Len()) m.data.All(func(key K, value V) bool { target[key] = value From 1d599faa551eeb62abf0bf5f83c75d38eeec4dd5 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sat, 8 Feb 2025 02:09:52 +0700 Subject: [PATCH 16/18] feat(swissmap): rm reads locker Signed-off-by: Dwi Siswanto --- swissmap/locker.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/swissmap/locker.go b/swissmap/locker.go index bedae10..c6be1d3 100644 --- a/swissmap/locker.go +++ b/swissmap/locker.go @@ -20,20 +20,20 @@ func (m *Map[K, V]) unlock() { } // rLock conditionally acquires the read lock if thread-safety is enabled -func (m *Map[K, V]) rLock() bool { - var locked bool - - if m.concurrent { - m.mutex.RLock() - locked = true - } - - return locked -} - -// rUnlock conditionally releases the read lock if thread-safety is enabled -func (m *Map[K, V]) rUnlock() { - if m.concurrent { - m.mutex.RUnlock() - } -} +// func (m *Map[K, V]) rLock() bool { +// var locked bool + +// if m.concurrent { +// m.mutex.RLock() +// locked = true +// } + +// return locked +// } + +// // rUnlock conditionally releases the read lock if thread-safety is enabled +// func (m *Map[K, V]) rUnlock() { +// if m.concurrent { +// m.mutex.RUnlock() +// } +// } From b1ea180ac7077040b6cb34866a492584ec8b1a82 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sat, 8 Feb 2025 02:10:52 +0700 Subject: [PATCH 17/18] chore(swissmap): add `.gitignore` Signed-off-by: Dwi Siswanto --- swissmap/.gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 swissmap/.gitignore diff --git a/swissmap/.gitignore b/swissmap/.gitignore new file mode 100644 index 0000000..edab275 --- /dev/null +++ b/swissmap/.gitignore @@ -0,0 +1,3 @@ +*.txt +*.out +*.test \ No newline at end of file From 1ce1825c3ca3c93513111d00faa8079e5e7c7dac Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sat, 8 Feb 2025 02:11:30 +0700 Subject: [PATCH 18/18] chore(swissmap): add bench script & update test Signed-off-by: Dwi Siswanto --- swissmap/bench.sh | 10 ++ swissmap/swissmap_bench_test.go | 193 +++++++++++++++++++++++--------- 2 files changed, 147 insertions(+), 56 deletions(-) create mode 100644 swissmap/bench.sh diff --git a/swissmap/bench.sh b/swissmap/bench.sh new file mode 100644 index 0000000..e171da3 --- /dev/null +++ b/swissmap/bench.sh @@ -0,0 +1,10 @@ +ITEMS=0 +if [ "$1" != "" ]; then + ITEMS=$1 +fi +rm -rf *bench.out +go test -run - -count=10 -timeout=1h -benchmem -bench . -items="$ITEMS" | \ + tee bench.out +grep -v "swissmap" bench.out | grep -v "BenchmarkHelper" | sed "s|/impl=mapsutil||g" > mapsutil-bench.out +grep -v "mapsutil" bench.out | grep -v "BenchmarkHelper" | sed "s|/impl=swissmap||g" > swissmap-bench.out +benchstat swissmap-bench.out mapsutil-bench.out \ No newline at end of file diff --git a/swissmap/swissmap_bench_test.go b/swissmap/swissmap_bench_test.go index 9a8588b..4cca212 100644 --- a/swissmap/swissmap_bench_test.go +++ b/swissmap/swissmap_bench_test.go @@ -1,13 +1,19 @@ package swissmap import ( + "flag" "fmt" "testing" + + mapsutil "github.com/projectdiscovery/utils/maps" ) -var benchNumItems = []int{1000, 5000, 10_000, 100_000, 250_000, 500_000, 1_000_000} +var ( + benchNumItems = []int{1, 1000, 100_000, 250_000, 500_000, 1_000_000} + items = flag.Int("items", 0, "run benchmarks with specific number of items") +) -func createMaps[K ComparableOrdered, V any](numItems int, threadSafe bool) *Map[K, V] { +func createMaps[K ComparableOrdered, V any](numItems int, threadSafe bool, ordered bool) *Map[K, V] { options := []Option[K, V]{ WithCapacity[K, V](numItems), } @@ -16,112 +22,187 @@ func createMaps[K ComparableOrdered, V any](numItems int, threadSafe bool) *Map[ options = append(options, WithConcurrentAccess[K, V]()) } + if ordered { + options = append(options, WithSortMapKeys[K, V]()) + } + return New[K, V](options...) } +func BenchmarkHelper(b *testing.B) { + if *items > 0 { + benchNumItems = []int{*items} + } + b.Helper() +} + func BenchmarkGet(b *testing.B) { for _, numItems := range benchNumItems { b.Run(fmt.Sprintf("items=%d", numItems), func(b *testing.B) { - m := createMaps[string, int](numItems, false) + m1 := createMaps[string, int](numItems, false, false) + m2 := mapsutil.Map[string, int]{} + for i := 0; i < numItems; i++ { - m.Set(fmt.Sprint(i), i) + m1.Set(fmt.Sprint(i), i) + m2.Set(fmt.Sprint(i), i) } b.ResetTimer() - b.Run("seq", func(b *testing.B) { + b.Run("impl=swissmap", func(b *testing.B) { + for i := 0; i < b.N; i++ { + m1.Get(fmt.Sprint(i % numItems)) + } + }) + + b.ResetTimer() + b.Run("impl=mapsutil", func(b *testing.B) { for i := 0; i < b.N; i++ { - m.Get(fmt.Sprint(i % numItems)) + m2.Get(fmt.Sprint(i % numItems)) } }) + }) + } +} + +func BenchmarkGet_Sorted(b *testing.B) { + for _, numItems := range benchNumItems { + b.Run(fmt.Sprintf("items=%d", numItems), func(b *testing.B) { + m1 := createMaps[string, int](numItems, false, true) + m2 := mapsutil.NewOrderedMap[string, int]() + + for i := 0; i < numItems; i++ { + m1.Set(fmt.Sprint(i), i) + m2.Set(fmt.Sprint(i), i) + } b.ResetTimer() - b.Run("parallel", func(b *testing.B) { + b.Run("impl=swissmap", func(b *testing.B) { + for i := 0; i < b.N; i++ { + m1.Get(fmt.Sprint(i % numItems)) + } + }) + + b.ResetTimer() + b.Run("impl=mapsutil", func(b *testing.B) { + for i := 0; i < b.N; i++ { + m2.Get(fmt.Sprint(i % numItems)) + } + }) + }) + } +} + +func BenchmarkGet_Sync(b *testing.B) { + for _, numItems := range benchNumItems { + b.Run(fmt.Sprintf("items=%d", numItems), func(b *testing.B) { + m1 := createMaps[string, int](numItems, true, false) + m2 := mapsutil.NewSyncLockMap[string, int]() + + for i := 0; i < numItems; i++ { + m1.Set(fmt.Sprint(i), i) + m2.Set(fmt.Sprint(i), i) + } + + b.ResetTimer() + b.Run("impl=swissmap", func(b *testing.B) { b.RunParallel(func(pb *testing.PB) { i := 0 for pb.Next() { - _, _ = m.Get(fmt.Sprint(i % numItems)) + m1.Get(fmt.Sprint(i % numItems)) i++ } }) }) - b.Run("WithConcurrentAccess", func(b *testing.B) { - m := createMaps[string, int](numItems, true) - for i := 0; i < numItems; i++ { - m.Set(fmt.Sprint(i), i) - } - - b.ResetTimer() - b.Run("seq", func(b *testing.B) { - for i := 0; i < b.N; i++ { - _, _ = m.Get(fmt.Sprint(i % numItems)) + b.ResetTimer() + b.Run("impl=mapsutil", func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + m2.Get(fmt.Sprint(i % numItems)) + i++ } }) - - b.ResetTimer() - b.Run("parallel", func(b *testing.B) { - b.RunParallel(func(pb *testing.PB) { - i := 0 - for pb.Next() { - _, _ = m.Get(fmt.Sprint(i % numItems)) - i++ - } - }) - }) }) }) } } +// ----------------------------------------------------------------------------- + func BenchmarkSet(b *testing.B) { for _, numItems := range benchNumItems { b.Run(fmt.Sprintf("items=%d", numItems), func(b *testing.B) { - m := createMaps[string, int](numItems, false) + m1 := createMaps[string, int](numItems, false, false) + m2 := mapsutil.Map[string, int]{} + + b.ResetTimer() + b.Run("impl=swissmap", func(b *testing.B) { + for i := 0; i < b.N; i++ { + m1.Set(fmt.Sprint(i), i) + } + }) b.ResetTimer() - b.Run("seq", func(b *testing.B) { + b.Run("impl=mapsutil", func(b *testing.B) { for i := 0; i < b.N; i++ { - m.Set(fmt.Sprint(i), i) + m2.Set(fmt.Sprint(i), i) } }) + }) + } +} - m.Clear() +func BenchmarkSet_Sorted(b *testing.B) { + for _, numItems := range benchNumItems { + b.Run(fmt.Sprintf("items=%d", numItems), func(b *testing.B) { + m1 := createMaps[string, int](numItems, false, true) + m2 := mapsutil.NewOrderedMap[string, int]() b.ResetTimer() - b.Run("parallel", func(b *testing.B) { + b.Run("impl=swissmap", func(b *testing.B) { + for i := 0; i < b.N; i++ { + m1.Set(fmt.Sprint(i), i) + } + }) + + b.ResetTimer() + b.Run("impl=mapsutil", func(b *testing.B) { + for i := 0; i < b.N; i++ { + m2.Set(fmt.Sprint(i), i) + } + }) + }) + } +} + +func BenchmarkSet_Sync(b *testing.B) { + for _, numItems := range benchNumItems { + b.Run(fmt.Sprintf("items=%d", numItems), func(b *testing.B) { + m1 := createMaps[string, int](numItems, true, false) + m2 := mapsutil.NewSyncLockMap[string, int]() + + b.ResetTimer() + b.Run("impl=swissmap", func(b *testing.B) { b.RunParallel(func(pb *testing.PB) { i := 0 for pb.Next() { - m.Set(fmt.Sprint(i), i) + m1.Set(fmt.Sprint(i), i) i++ } }) }) - b.Run("WithThreadSafety", func(b *testing.B) { - m := createMaps[string, int](numItems, true) - - b.ResetTimer() - b.Run("seq", func(b *testing.B) { - for i := 0; i < b.N; i++ { - m.Set(fmt.Sprint(i), i) + b.ResetTimer() + b.Run("impl=mapsutil", func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + m2.Set(fmt.Sprint(i), i) + i++ } }) - - m.Clear() - - b.ResetTimer() - b.Run("parallel", func(b *testing.B) { - b.RunParallel(func(pb *testing.PB) { - i := 0 - for pb.Next() { - m.Set(fmt.Sprint(i), i) - i++ - } - }) - }) }) - }) } }