feature: introduces FillUp method

Author: aliszka

benchmark_opt_test.go

@@ -32,6 +32,51 @@ func BenchmarkPrefillFromSortedList(b *testing.B) {
 	}
 }
 
+// go test -v -bench BenchmarkFillUpNative -benchmem -run ^$ github.com/weaviate/sroar -cpuprofile cpu.prof
+func BenchmarkFillUpNative(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		bm := Prefill(100_000_000)
+		bm.FillUp(150_000_000)
+		bm.FillUp(200_000_000)
+	}
+}
+
+// go test -v -bench BenchmarkPrefillFromSortedList -benchmem -run ^$ github.com/weaviate/sroar -cpuprofile cpu.prof
+func BenchmarkFillUpFromSortedList(b *testing.B) {
+	prefillBufferSize := 65_536
+	maxVal1 := uint64(100_000_000)
+	maxVal2 := uint64(150_000_000)
+	maxVal3 := uint64(200_000_000)
+	inc := uint64(prefillBufferSize)
+	buf := make([]uint64, prefillBufferSize)
+
+	for i := 0; i < b.N; i++ {
+		finalBM := NewBitmap()
+
+		for i := uint64(0); i <= maxVal1; i += inc {
+			j := uint64(0)
+			for ; j < inc && i+j <= maxVal1; j++ {
+				buf[j] = i + j
+			}
+			finalBM.Or(FromSortedList(buf[:j]))
+		}
+		for i := maxVal1 + 1; i <= maxVal2; i += inc {
+			j := uint64(0)
+			for ; j < inc && i+j <= maxVal2; j++ {
+				buf[j] = i + j
+			}
+			finalBM.Or(FromSortedList(buf[:j]))
+		}
+		for i := maxVal2 + 1; i <= maxVal3; i += inc {
+			j := uint64(0)
+			for ; j < inc && i+j <= maxVal3; j++ {
+				buf[j] = i + j
+			}
+			finalBM.Or(FromSortedList(buf[:j]))
+		}
+	}
+}
+
 // ================================================================================
 //
 // BENCHMARKS comparing performance of different merge implementations

bitmap.go

@@ -115,7 +115,6 @@ func newBitampToBuf(keysLen, initialContainerSize int, buf []uint16) *Bitmap {
 	ra := &Bitmap{data: buf[:keysLen]}
 	ra.keys = toUint64Slice(ra.data)
 	ra.keys.setNodeSize(keysLen)
-
 	// Always generate a container for key = 0x00. Otherwise, node gets confused
 	// about whether a zero key is a new key or not.
 	offset := ra.newContainer(uint16(initialContainerSize))
@@ -244,9 +243,14 @@ func (ra *Bitmap) scootLeft(offset uint64, size uint64) {
 }
 
 func (ra *Bitmap) newContainer(sz uint16) uint64 {
+	// fmt.Printf("  ==> newContainer sz [%d]\n", sz)
 	offset := ra.newContainerNoClr(sz)
+	// fmt.Printf("  ==> newContainer offset [%d]\n", offset)
 	ra.data[offset] = sz
+	// fmt.Printf("  ==> newContainer ra.data[offset] [%d]\n", ra.data[offset])
 	Memclr(ra.data[offset+1 : offset+uint64(sz)])
+	// fmt.Printf("  ==> newContainer ra.data[offset] [%d]\n", ra.data[offset])
+
 	return offset
 }