Instant query optimization v2

model/labels/labels_dedupelabels_struct_size.go

@@ -0,0 +1,18 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build dedupelabels
+
+package labels
+
+const LabelsStructSize uint8 = 24

model/labels/labels_struct_size.go

@@ -0,0 +1,18 @@
+// Copyright 2017 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !stringlabels && !dedupelabels
+
+package labels
+
+const LabelsStructSize uint8 = 24

model/labels/labels_struct_size_stringlabels.go

@@ -0,0 +1,18 @@
+// Copyright 2017 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build stringlabels
+
+package labels
+
+const LabelsStructSize uint8 = 16

pp/entrypoint/go_constants.h

@@ -3,5 +3,6 @@
 #define Sizeof_BareBonesVector 16
 #define Sizeof_RoaringBitset 40
 #define Sizeof_InnerSeries (Sizeof_SizeT + Sizeof_BareBonesVector + Sizeof_RoaringBitset)
+#define Sizeof_GoLabels 16
 
 #define Sizeof_SerializedDataIterator 192

pp/entrypoint/series_data/querier.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "bare_bones/bitset.h"
+#include "entrypoint/go_constants.h"
 #include "primitives/go_slice.h"
 #include "primitives/primitives.h"
 #include "series_data/querier/instant_querier.h"
@@ -47,8 +48,13 @@ class InstantQuerierWithArgumentsWrapper {
   const Timestamp timestamp_;
 };
 
+struct SampleWithGoLabels : public ::series_data::encoder::Sample {
+ private:
+  char go_labels_[Sizeof_GoLabels];
+};
+
 using InstantQuerierWithArgumentsWrapperEntrypoint = InstantQuerierWithArgumentsWrapper<PromPP::Primitives::Go::SliceView<PromPP::Primitives::LabelSetID>,
-                                                                                        PromPP::Primitives::Go::SliceView<::series_data::encoder::Sample>>;
+                                                                                        std::span<entrypoint::series_data::SampleWithGoLabels>>;
 
 class RangeQuerierWithArgumentsWrapper {
   using DataStorage = ::series_data::DataStorage;

pp/entrypoint/series_data_data_storage.cpp

@@ -177,13 +177,12 @@ extern "C" void prompp_series_data_data_storage_instant_query(void* args, void*
   using entrypoint::series_data::InstantQuerierWithArgumentsWrapperEntrypoint;
   using PromPP::Primitives::Timestamp;
   using series_data::InstantQuerier;
-  using series_data::encoder::Sample;
 
   struct Arguments {
     DataStoragePtr data_storage;
     SliceView<LabelSetID> label_set_ids;
     Timestamp timestamp;
-    SliceView<Sample> samples;
+    entrypoint::series_data::SampleWithGoLabels* samples;
   };
 
   using Result = struct {
@@ -193,7 +192,8 @@ extern "C" void prompp_series_data_data_storage_instant_query(void* args, void*
 
   const auto in = static_cast<Arguments*>(args);
 
-  InstantQuerierWithArgumentsWrapperEntrypoint instant_querier(*in->data_storage, in->label_set_ids, in->timestamp, in->samples);
+  auto samples = std::span<entrypoint::series_data::SampleWithGoLabels>(in->samples, in->label_set_ids.size());
+  InstantQuerierWithArgumentsWrapperEntrypoint instant_querier(*in->data_storage, in->label_set_ids, in->timestamp, samples);
   instant_querier.query();
 
   if (instant_querier.need_loading()) {

pp/entrypoint/series_data_data_storage.h

@@ -126,16 +126,14 @@ void prompp_series_data_data_storage_query(void* args, void* res);
 void prompp_series_data_data_storage_query_v2(void* args, void* res);
 
 /**
- * @brief return samples at given timestamp for label sets.
+ * @brief return instant series at given timestamp for label sets.
  *
  * @param args {
- *        dataStorage uintptr    // pointer to constructed data storage
- *        labelSetIDs []uint32   // series ids
- *        timestamp   int64      // timestamp
- *        samples     []struct { // pre-allocated samples slice
- *                timestamp int64
- *                value     float64
- *        }
+ *        dataStorage uintptr      // pointer to constructed data storage
+ *        labelSetIDs []uint32     // series ids
+ *        timestamp   int64        // timestamp
+ *        samples     uintptr      // pointer to samples data
+ * }
  * @param res {
  *     InstantQuerier uintptr // pointer to constructed Querier if data loading is needed
  *     Status uint8           // status of a query (0 - Success, 1 - Data loading is needed)

pp/go/cppbridge/data_storage.go

@@ -0,0 +1,121 @@
+package cppbridge
+
+import (
+	"runtime"
+)
+
+// DataStorage is Go wrapper around series_data::Data_storage.
+type DataStorage struct {
+	dataStorage       uintptr
+	gcDestroyDetector *uint64
+	timeInterval      TimeInterval
+}
+
+// NewDataStorage - constructor.
+func NewDataStorage() *DataStorage {
+	ds := &DataStorage{
+		dataStorage:       seriesDataDataStorageCtor(),
+		gcDestroyDetector: &gcDestroyDetector,
+		timeInterval:      NewInvalidTimeInterval(),
+	}
+
+	runtime.SetFinalizer(ds, func(ds *DataStorage) {
+		seriesDataDataStorageDtor(ds.dataStorage)
+	})
+
+	return ds
+}
+
+// Reset - resets data storage.
+func (ds *DataStorage) Reset() {
+	seriesDataDataStorageReset(ds.dataStorage)
+	ds.timeInterval = NewInvalidTimeInterval()
+}
+
+func (ds *DataStorage) TimeInterval(invalidateCache bool) TimeInterval {
+	if invalidateCache || ds.timeInterval.IsInvalid() {
+		ds.timeInterval = seriesDataDataStorageTimeInterval(ds.dataStorage)
+		runtime.KeepAlive(ds)
+	}
+
+	return ds.timeInterval
+}
+
+func (ds *DataStorage) GetQueriedSeriesBitset() []byte {
+	size := seriesDataDataStorageQueriedSeriesBitsetSize(ds.dataStorage)
+	bitset := seriesDataDataStorageQueriedSeriesBitset(ds.dataStorage, make([]byte, 0, size))
+	runtime.KeepAlive(ds)
+	return bitset
+}
+
+func (ds *DataStorage) SetQueriedSeriesBitset(bitset []byte) bool {
+	result := seriesDataDataStorageQueriedSeriesSetBitset(ds.dataStorage, bitset)
+	runtime.KeepAlive(ds)
+	return result
+}
+
+func (ds *DataStorage) Pointer() uintptr {
+	return ds.dataStorage
+}
+
+func (ds *DataStorage) AllocatedMemory() uint64 {
+	res := seriesDataDataStorageAllocatedMemory(ds.dataStorage)
+	runtime.KeepAlive(ds)
+	return res
+}
+
+type UnusedSeriesDataUnloader struct {
+	unloader uintptr
+	ds       *DataStorage
+}
+
+func (u *UnusedSeriesDataUnloader) CreateSnapshot() []byte {
+	snapshot := seriesDataUnusedSeriesDataUnloaderCreateSnapshot(u.unloader)
+	runtime.KeepAlive(u)
+	return snapshot
+}
+
+func (u *UnusedSeriesDataUnloader) Unload() {
+	seriesDataUnusedSeriesDataUnloaderUnload(u.unloader)
+	runtime.KeepAlive(u)
+}
+
+func (ds *DataStorage) CreateUnusedSeriesDataUnloader() *UnusedSeriesDataUnloader {
+	unloader := &UnusedSeriesDataUnloader{
+		unloader: seriesDataUnusedSeriesDataUnloaderCtor(ds.dataStorage),
+		ds:       ds,
+	}
+
+	runtime.SetFinalizer(unloader, func(u *UnusedSeriesDataUnloader) {
+		seriesDataUnusedSeriesDataUnloaderDtor(u.unloader)
+	})
+
+	return unloader
+}
+
+type DataStorageQuery struct {
+	StartTimestampMs int64
+	EndTimestampMs   int64
+	LabelSetIDs      []uint32
+}
+
+func (ds *DataStorage) Query(query DataStorageQuery) DataStorageQueryResult {
+	sd := NewDataStorageSerializedData()
+	querier, status := seriesDataDataStorageQueryV2(ds.dataStorage, query, sd)
+	return DataStorageQueryResult{
+		Querier:        querier,
+		Status:         status,
+		SerializedData: sd,
+	}
+}
+
+// InstantQuery .
+// Deprecated: InstantQuery .
+func (ds *DataStorage) InstantQuery(targetTimestamp int64, labelSetIDs []uint32, samples uintptr) DataStorageQueryResult {
+	return seriesDataDataStorageInstantQuery(ds.dataStorage, labelSetIDs, targetTimestamp, samples)
+}
+
+func (ds *DataStorage) QueryFinal(queriers []uintptr) {
+	seriesDataDataStorageQueryFinal(queriers)
+	runtime.KeepAlive(queriers)
+}

pp/go/cppbridge/entrypoint.go

@@ -34,6 +34,10 @@ type (
 	CppSerializedDataIterator = [C.Sizeof_SerializedDataIterator]byte
 )
 
+const (
+	GoLabelsSize = C.Sizeof_GoLabels
+)
+
 var (
 
 	// per_goroutine_relabeler input_relabeling
@@ -1856,10 +1860,10 @@ type DataStorageQueryResult struct {
 	SerializedData *DataStorageSerializedData
 }
 
-func seriesDataDataStorageQueryV2(dataStorage uintptr, query HeadDataStorageQuery, serializedData *DataStorageSerializedData) (querier uintptr, status uint8) {
+func seriesDataDataStorageQueryV2(dataStorage uintptr, query DataStorageQuery, serializedData *DataStorageSerializedData) (querier uintptr, status uint8) {
 	args := struct {
 		dataStorage uintptr
-		query       HeadDataStorageQuery
+		query       DataStorageQuery
 	}{dataStorage, query}
 
 	var res = struct {
@@ -1883,14 +1887,13 @@ func seriesDataDataStorageQueryV2(dataStorage uintptr, query HeadDataStorageQuer
 	return res.Querier, res.Status
 }
 
-func seriesDataDataStorageInstantQuery(dataStorage uintptr, labelSetIDs []uint32, timestamp int64, samples []Sample) DataStorageQueryResult {
+func seriesDataDataStorageInstantQuery(dataStorage uintptr, labelSetIDs []uint32, timestamp int64, samples uintptr) DataStorageQueryResult {
 	args := struct {
 		dataStorage uintptr
 		labelSetIDs []uint32
 		timestamp   int64
-		samples     []Sample
+		samples     uintptr
 	}{dataStorage, labelSetIDs, timestamp, samples}
-
 	var res DataStorageQueryResult
 
 	testGC()

pp/go/cppbridge/entrypoint.h

@@ -38,6 +38,7 @@ void prompp_dump_memory_profile(void* args, void* res);
 #define Sizeof_BareBonesVector 16
 #define Sizeof_RoaringBitset 40
 #define Sizeof_InnerSeries (Sizeof_SizeT + Sizeof_BareBonesVector + Sizeof_RoaringBitset)
+#define Sizeof_GoLabels 16
 
 #define Sizeof_SerializedDataIterator 192
 #ifdef __cplusplus
@@ -1361,16 +1362,14 @@ void prompp_series_data_data_storage_query(void* args, void* res);
 void prompp_series_data_data_storage_query_v2(void* args, void* res);
 
 /**
- * @brief return samples at given timestamp for label sets.
+ * @brief return instant series at given timestamp for label sets.
  *
  * @param args {
- *        dataStorage uintptr    // pointer to constructed data storage
- *        labelSetIDs []uint32   // series ids
- *        timestamp   int64      // timestamp
- *        samples     []struct { // pre-allocated samples slice
- *                timestamp int64
- *                value     float64
- *        }
+ *        dataStorage uintptr      // pointer to constructed data storage
+ *        labelSetIDs []uint32     // series ids
+ *        timestamp   int64        // timestamp
+ *        samples     uintptr      // pointer to samples data
+ * }
  * @param res {
  *     InstantQuerier uintptr // pointer to constructed Querier if data loading is needed
  *     Status uint8           // status of a query (0 - Success, 1 - Data loading is needed)