feature: introduces FillUp method

benchmark_opt_test.go

@@ -32,6 +32,44 @@ func BenchmarkPrefillFromSortedList(b *testing.B) {
 	}
 }
 
+// go test -v -bench BenchmarkFillUpNative -benchmem -run ^$ github.com/weaviate/sroar -cpuprofile cpu.prof
+func BenchmarkFillUpNative(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		bm := Prefill(100_000_000)
+		bm.FillUp(150_000_000)
+		bm.FillUp(200_000_000)
+	}
+}
+
+// go test -v -bench BenchmarkPrefillFromSortedList -benchmem -run ^$ github.com/weaviate/sroar -cpuprofile cpu.prof
+func BenchmarkFillUpFromSortedList(b *testing.B) {
+	prefillBufferSize := 65_536
+	prefillX := uint64(100_000_000)
+	fillupX1 := uint64(150_000_000)
+	fillupX2 := uint64(200_000_000)
+	inc := uint64(prefillBufferSize)
+	buf := make([]uint64, prefillBufferSize)
+
+	for i := 0; i < b.N; i++ {
+		bm := Prefill(prefillX)
+
+		for i := prefillX + 1; i <= fillupX1; i += inc {
+			j := uint64(0)
+			for ; j < inc && i+j <= fillupX1; j++ {
+				buf[j] = i + j
+			}
+			bm.Or(FromSortedList(buf[:j]))
+		}
+		for i := fillupX1 + 1; i <= fillupX2; i += inc {
+			j := uint64(0)
+			for ; j < inc && i+j <= fillupX2; j++ {
+				buf[j] = i + j
+			}
+			bm.Or(FromSortedList(buf[:j]))
+		}
+	}
+}
+
 // ================================================================================
 //
 // BENCHMARKS comparing performance of different merge implementations

bitmap_opt.go

@@ -710,84 +710,291 @@ func (ra *Bitmap) CloneToBuf(buf []byte) *Bitmap {
 
 // Prefill creates bitmap prefilled with elements [0-maxX]
 func Prefill(maxX uint64) *Bitmap {
-	maxCard64 := uint64(maxCardinality)
+	containersCount, remainingCount := calcFullContainersAndRemainingCounts(maxX)
 
-	// maxX should be included, therefore +1
-	n := maxX / maxCard64
-	rem := maxX % maxCard64
-	if rem == maxCard64-1 {
-		n++
-	}
-	rem = (rem + 1) % maxCard64
-
-	// create additional container for remaining values (or empty if there are not)
+	// create additional container for remaining values
+	// (or reserve space for new one if there are not any remaining)
 	// +1 additional key to avoid keys expanding (there should always be 1 spare)
-	bm := newBitmapWith(int(n)+1+1, maxContainerSize, int(n)*maxContainerSize)
+	bm := newBitmapWith(int(containersCount)+1+1, maxContainerSize, int(containersCount)*maxContainerSize)
+	bm.prefill(containersCount, remainingCount)
+	return bm
+}
 
-	var refContainer []uint16
-	var remOffset = bm.keys.val(0)
+func (ra *Bitmap) FillUp(maxX uint64) {
+	if ra == nil {
+		return
+	}
 
-	if n > 0 {
-		refContainer = bm.getContainer(remOffset)
-		refContainer[indexSize] = maxContainerSize
-		refContainer[indexType] = typeBitmap
-		setCardinality(refContainer, maxCardinality)
+	maxContainersCount, maxRemainingCount := calcFullContainersAndRemainingCounts(maxX)
+	if ra.IsEmpty() {
+		// try to fit data into existing memory,
+		// if there is not enough space anyway, allocate more memory to fit additional container
+		minimalContainersCount := maxContainersCount
+		if maxRemainingCount > 0 {
+			minimalContainersCount++
+		}
+		minimalKeysLen := calcInitialKeysLen(minimalContainersCount + 1)
+		minimalLen := minimalKeysLen + minimalContainersCount*maxContainerSize
 
-		// fill entire bitmap container with ones
-		refContainer64 := uint16To64SliceUnsafe(refContainer[startIdx:])
-		for i := range refContainer64 {
-			refContainer64[i] = math.MaxUint64
+		var bm *Bitmap
+		if minimalLen <= cap(ra.data) {
+			bm = newBitampToBuf(minimalKeysLen, maxContainerSize, ra.data)
+		} else {
+			bm = newBitmapWith(int(maxContainersCount)+1+1, maxContainerSize, int(maxContainersCount)*maxContainerSize)
 		}
+		bm.prefill(maxContainersCount, maxRemainingCount)
+		ra.data = bm.data
+		ra._ptr = bm._ptr
+		ra.keys = bm.keys
+		return
+	}
 
-		// fill remaining containers by copying reference one
-		for i := uint64(1); i < n; i++ {
-			key := (i * maxCard64) & mask
-			offset := bm.newContainerNoClr(maxContainerSize)
-			bm.setKey(key, offset)
+	minX := ra.Maximum()
+	if minX >= maxX {
+		return
+	}
 
-			copy(bm.data[offset:], refContainer)
+	maxKey := maxX & mask
+	minKey := minX & mask
+	maxY := int(uint16(maxX))
+	minY := int(uint16(minX))
+
+	idx := ra.keys.searchReverse(minKey)
+	minOffset := ra.keys.val(idx)
+
+	// same container
+	if maxKey == minKey {
+		commonContainer := ra.getContainer(minOffset)
+		card := getCardinality(commonContainer)
+		newYs := maxY - minY
+
+		switch commonContainer[indexType] {
+		case typeBitmap:
+			// set bits in bitmap
+			bitmap(commonContainer).setRange(minY, maxY, nil)
+			setCardinality(commonContainer, card+newYs)
+		case typeArray:
+			size := commonContainer[indexSize]
+			if spaceLeft := int(size-startIdx) - card; spaceLeft >= newYs {
+				// add elements to existing array
+				for i := 0; i < newYs; i++ {
+					commonContainer[startIdx+uint16(card+i)] = uint16(minY + 1 + i)
+				}
+				setCardinality(commonContainer, card+newYs)
+			} else {
+				// create new bitmap container, copy elements from array, set new bits
+				prevContainer := commonContainer
+				minOffset = ra.newContainer(maxContainerSize)
+				ra.setKey(minKey, minOffset)
+
+				commonContainer = ra.fillUpBitmapContainerRange(minOffset, minY, maxY, card+newYs, nil)
+				for i := 0; i < card; i++ {
+					y := prevContainer[startIdx+uint16(i)]
+					commonContainer[startIdx+y/16] |= bitmapMask[y%16]
+				}
+			}
+		default:
+			panic("unknown container type")
 		}
+		return
+	}
 
-		// create container for remaining values
-		key := (n * maxCard64) & mask
-		remOffset = bm.newContainer(maxContainerSize)
-		bm.setKey(key, remOffset)
+	minContainersCount, minRemainingCount := calcFullContainersAndRemainingCounts(minX)
+	requiredContainersCount := maxContainersCount - minContainersCount
+	if maxRemainingCount > 0 {
+		requiredContainersCount++
 	}
 
-	container := bm.getContainer(remOffset)
-	container[indexSize] = maxContainerSize
-	container[indexType] = typeBitmap
-	setCardinality(container, int(rem))
+	// first count how many new containers will be required, to allocate memory once, then do the fillup
+	var fillUpCommonContainer func(commonContainer []uint16, onesBitmap bitmap) = nil
+	// idx of first full container to add
+	containerIdx := minContainersCount
+	if minRemainingCount > 0 {
+		containerIdx++
+		requiredContainersCount--
 
-	if rem > 0 {
-		n16 := uint16(rem) / 16
-		rem16 := uint16(rem) % 16
+		commonContainer := ra.getContainer(minOffset)
+		card := getCardinality(commonContainer)
+		newYs := maxCardinality - 1 - minY
 
-		if refContainer != nil {
-			// refContainer available (maxX >= math.MaxUint16-1),
-			// fill remaining values container by copying biggest possible slice of refContainer (batches of 16s)
-			copy(bm.data[remOffset+uint64(startIdx):], refContainer[startIdx:startIdx+n16])
-			// set remaining bits
-			for i := uint16(0); i < rem16; i++ {
-				container[startIdx+n16] |= bitmapMask[i]
+		switch commonContainer[indexType] {
+		case typeBitmap:
+			// if bitmap, set proper bits up to maxCardinality
+			fillUpCommonContainer = func(commonContainer []uint16, onesBitmap bitmap) {
+				bitmap(commonContainer).setRange(minY, maxCardinality-1, onesBitmap)
+				setCardinality(commonContainer, card+newYs)
 			}
+		case typeArray:
+			size := commonContainer[indexSize]
+			if spaceLeft := int(size-startIdx) - card; spaceLeft >= newYs {
+				// if array add new elements if there is enough space left
+				fillUpCommonContainer = func(commonContainer []uint16, onesBitmap bitmap) {
+					for i := 0; i < newYs; i++ {
+						commonContainer[startIdx+uint16(card+i)] = uint16(minY + 1 + i)
+					}
+					setCardinality(commonContainer, card+newYs)
+				}
+			} else {
+				// if not enough space, create new bitmap container, set new bits and set old ones
+				requiredContainersCount++
+				fillUpCommonContainer = func(commonContainer []uint16, onesBitmap bitmap) {
+					prevContainer := commonContainer
+					offset := ra.newContainer(maxContainerSize)
+					ra.setKey(minKey, offset)
+
+					commonContainer = ra.fillUpBitmapContainerRange(offset, minY, maxCardinality-1, card+newYs, onesBitmap)
+					for i := 0; i < card; i++ {
+						y := prevContainer[startIdx+uint16(i)]
+						commonContainer[startIdx+y/16] |= bitmapMask[y%16]
+					}
+				}
+			}
+		default:
+			panic("unknown container type")
+		}
+	}
+
+	// calculate required memory to allocate and expand underlying slice
+	containersLen := uint64(requiredContainersCount * maxContainerSize)
+	keysLen := uint64(requiredContainersCount * 2 * 4)
+	ra.expandNoLengthChange(containersLen + keysLen)
+	ra.expandKeys(keysLen)
+
+	var onesBitmap bitmap
+	if containerIdx < maxContainersCount {
+		// fillup full containers
+		key := uint64(containerIdx*maxCardinality) & mask
+		offset := ra.newContainerNoClr(maxContainerSize)
+		ra.setKey(key, offset)
+
+		onesBitmap = ra.fillUpBitmapContainers(offset, containerIdx+1, maxContainersCount)
+	}
+	if maxRemainingCount > 0 {
+		// fillup last container
+		key := uint64(maxContainersCount*maxCardinality) & mask
+		offset := ra.newContainer(maxContainerSize)
+		ra.setKey(key, offset)
+
+		ra.fillUpBitmapContainerRange(offset, 0, maxRemainingCount-1, maxRemainingCount, onesBitmap)
+	}
+	if minRemainingCount > 0 {
+		// fillup common container using previously created callback
+		// due to slice expanding, container has to be fetched once again due to new offset
+		minOffset = ra.keys.val(idx)
+		commonContainer := ra.getContainer(minOffset)
+		fillUpCommonContainer(commonContainer, onesBitmap)
+	}
+}
+
+func (ra *Bitmap) prefill(containersCount, remainingCount int) {
+	var onesBitmap bitmap
+
+	if containersCount > 0 {
+		offset := ra.keys.val(0)
+		onesBitmap = ra.fillUpBitmapContainers(offset, 1, containersCount)
+	}
+	if remainingCount > 0 {
+		var offset uint64
+		if containersCount > 0 {
+			// create container for remaining values
+			key := uint64(containersCount*maxCardinality) & mask
+			offset = ra.newContainer(maxContainerSize)
+			ra.setKey(key, offset)
 		} else {
-			// refContainer not available (maxX < math.MaxUint16-1),
-			// set bits by copying MaxUint64 first, then MaxUint16, then single bits
-			n64 := uint16(rem) / 64
+			// get initial container
+			offset = ra.keys.val(0)
+		}
+		ra.fillUpBitmapContainerRange(offset, 0, remainingCount-1, remainingCount, onesBitmap)
+	}
+}
+
+func (ra *Bitmap) fillUpBitmapContainerRange(offset uint64, minY, maxY, card int, onesBitmap bitmap) bitmap {
+	bitmap := bitmap(ra.getContainer(offset))
+	bitmap[indexSize] = maxContainerSize
+	bitmap[indexType] = typeBitmap
+	setCardinality(bitmap, card)
+
+	bitmap.setRange(minY, maxY, onesBitmap)
+	return bitmap
+}
+
+func (ra *Bitmap) fillUpBitmapContainers(offset uint64, minIdx, maxIdx int) bitmap {
+	onesBitmap := bitmap(ra.getContainer(offset)[:maxContainerSize])
+	onesBitmap[indexSize] = maxContainerSize
+	onesBitmap[indexType] = typeBitmap
+	setCardinality(onesBitmap, maxCardinality)
+
+	// fill entire bitmap container with ones
+	onesBitmap.fillWithOnes()
+
+	// fill remaining containers by copying first one
+	for i := minIdx; i < maxIdx; i++ {
+		key := uint64(i*maxCardinality) & mask
+		offset := ra.newContainerNoClr(maxContainerSize)
+		ra.setKey(key, offset)
+		copy(ra.data[offset:], onesBitmap)
+	}
+	return onesBitmap
+}
+
+func calcFullContainersAndRemainingCounts(maxX uint64) (int, int) {
+	maxCard64 := uint64(maxCardinality)
+
+	// maxX should be included, therefore +1
+	containers := maxX / maxCard64
+	remaining := maxX % maxCard64
+	if remaining == maxCard64-1 {
+		containers++
+	}
+	remaining = (remaining + 1) % maxCard64
+	return int(containers), int(remaining)
+}
+
+func (b bitmap) setRange(leftY, rightY int, onesBitmap bitmap) {
+	leftY16 := (leftY + 15) / 16 * 16
+	rightY16 := (rightY + 1) / 16 * 16
+
+	// fmt.Printf("  ==> left [%d] left16 [%d] left64 [%d]\n", leftY, leftY16, (leftY+63)/64*64)
+	// fmt.Printf("  ==> right [%d] right16 [%d] right64 [%d]\n", rightY, rightY16, (rightY+1)/64*64)
 
-			container64 := uint16To64SliceUnsafe(container[startIdx:])
-			for i := uint16(0); i < n64; i++ {
+	container16 := b[startIdx:]
+	if onesBitmap != nil {
+		if l, r := uint16(leftY16/16), uint16(rightY16/16); l < r {
+			copy(container16[l:r], onesBitmap[startIdx+l:startIdx+r])
+		}
+	} else {
+		leftY64 := (leftY + 63) / 64 * 64
+		rightY64 := (rightY + 1) / 64 * 64
+
+		if l, r := leftY64/64, rightY64/64; l < r {
+			container64 := uint16To64SliceUnsafe(container16)
+			for i := l; i < r; i++ {
+				// fmt.Printf("    ==> container64 i=%d\n", i)
 				container64[i] = math.MaxUint64
 			}
-			for i := uint16(n64 * 4); i < n16; i++ {
-				container[startIdx+i] = math.MaxUint16
-			}
-			for i := uint16(0); i < rem16; i++ {
-				container[startIdx+n16] |= bitmapMask[i]
-			}
 		}
+		for i, r := leftY16/16, min(leftY64/16, rightY16/16); i < r; i++ {
+			// fmt.Printf("    ==> container16L i=%d\n", i)
+			container16[i] = math.MaxUint16
+		}
+		for i, r := max(leftY16/16, rightY64/16), rightY16/16; i < r; i++ {
+			// fmt.Printf("    ==> container16R i=%d\n", i)
+			container16[i] = math.MaxUint16
+		}
+	}
+	for y, r := leftY, min(leftY16, rightY+1); y < r; y++ {
+		// fmt.Printf("    ==> container16L i=%d bit=%d\n", y/16, y%16)
+		container16[y/16] |= bitmapMask[y%16]
 	}
+	for y, r := max(leftY, rightY16), rightY+1; y < r; y++ {
+		// fmt.Printf("    ==> container16R i=%d bit=%d\n", y/16, y%16)
+		container16[y/16] |= bitmapMask[y%16]
+	}
+}
 
-	return bm
+func (b bitmap) fillWithOnes() {
+	b64 := uint16To64SliceUnsafe(b[startIdx:])
+	for i := range b64 {
+		b64[i] = math.MaxUint64
+	}
 }