From ad6bbde6ce8ec1cfd336b2e9b8caa0436d36d3d5 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Thu, 9 Oct 2025 11:24:00 +0800 Subject: [PATCH 01/31] Pipe: Add tsFile parsing with Mods function --- .../TsFileInsertionEventQueryParser.java | 94 ++++- ...sertionEventQueryParserTabletIterator.java | 91 ++++- .../scan/TsFileInsertionEventScanParser.java | 331 ++++++++++++------ 3 files changed, 404 insertions(+), 112 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java index d61f7a791ca5..c0128381b5ed 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser.query; +import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; @@ -30,14 +31,19 @@ import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.read.TsFileDeviceIterator; import org.apache.tsfile.read.TsFileReader; import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.record.Tablet; import org.slf4j.Logger; @@ -54,6 +60,7 @@ import java.util.NoSuchElementException; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; public class TsFileInsertionEventQueryParser extends TsFileInsertionEventParser { @@ -67,6 +74,10 @@ public class TsFileInsertionEventQueryParser extends TsFileInsertionEventParser private final Map deviceIsAlignedMap; private final Map measurementDataTypeMap; + // mods entry + private PatternTreeMap currentModifications = + PatternTreeMapFactory.getModsPatternTreeMap(); + @TestOnly public TsFileInsertionEventQueryParser( final File tsFile, @@ -114,6 +125,12 @@ public TsFileInsertionEventQueryParser( super(pipeName, creationTime, pattern, null, startTime, endTime, pipeTaskMeta, sourceEvent); try { + // Read mods file first + ModificationFile.readAllModifications(tsFile, true) + .forEach( + modification -> + currentModifications.append(modification.keyOfPatternTree(), modification)); + final PipeTsFileResourceManager tsFileResourceManager = PipeDataNodeResourceManager.tsfile(); final Map> deviceMeasurementsMap; @@ -158,6 +175,61 @@ public TsFileInsertionEventQueryParser( allocatedMemoryBlock = PipeDataNodeResourceManager.memory().forceAllocate(memoryRequiredInBytes); + final Iterator>> iterator = + deviceMeasurementsMap.entrySet().iterator(); + while (iterator.hasNext()) { + final Map.Entry> entry = iterator.next(); + final IDeviceID deviceId = entry.getKey(); + final List measurements = entry.getValue(); + + // Check if deviceId is deleted + if (deviceId == null) { + LOGGER.warn("Found null deviceId, removing entry"); + iterator.remove(); + continue; + } + + // Check if measurements list is deleted or empty + if (measurements == null || measurements.isEmpty()) { + LOGGER.warn( + "Found null or empty measurements list for deviceId: {}, removing entry", deviceId); + iterator.remove(); + continue; + } + + // Safely filter measurements, remove non-existent measurements + measurements.removeIf( + measurement -> { + if (measurement == null || measurement.isEmpty()) { + LOGGER.warn("Found null or empty measurement for deviceId: {}, removing", deviceId); + return true; + } + + try { + TimeseriesMetadata meta = + tsFileSequenceReader.readTimeseriesMetadata(deviceId, measurement, true); + return isAllDeletedByMods( + deviceId, + measurement, + meta.getStatistics().getStartTime(), + meta.getStatistics().getEndTime()); + } catch (IOException e) { + LOGGER.warn( + "Failed to read metadata for deviceId: {}, measurement: {}, removing", + deviceId, + measurement, + e); + return true; + } + }); + + // If measurements list is empty after filtering, remove the entire entry + if (measurements.isEmpty()) { + LOGGER.warn("No valid measurements left for deviceId: {}, removing entry", deviceId); + iterator.remove(); + } + } + // Filter again to get the final deviceMeasurementsMap that exactly matches the pattern. deviceMeasurementsMapIterator = filterDeviceMeasurementsMapByPattern(deviceMeasurementsMap).entrySet().iterator(); @@ -303,7 +375,8 @@ public boolean hasNext() { entry.getKey(), entry.getValue(), timeFilterExpression, - allocatedMemoryBlockForTablet); + allocatedMemoryBlockForTablet, + currentModifications); } catch (final Exception e) { close(); throw new PipeException( @@ -407,4 +480,23 @@ public void close() { allocatedMemoryBlock.close(); } } + + public boolean isAllDeletedByMods( + IDeviceID deviceID, String measurementID, long startTime, long endTime) { + final List mods = currentModifications.getOverlapped(deviceID, measurementID); + return mods.stream() + .anyMatch( + modification -> + modification.getTimeRange().contains(startTime, endTime) + && (!deviceID.isTableModel() || modification.affects(deviceID))); + } + + public List getModTimeRanges(IDeviceID deviceID, String measurementID) { + final List mods = currentModifications.getOverlapped(deviceID, measurementID); + return mods.stream() + .filter(modification -> (!deviceID.isTableModel() || modification.affects(deviceID))) + .map(ModEntry::getTimeRange) + .sorted() + .collect(Collectors.toList()); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java index 7c32321186e3..5024414b0399 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java @@ -19,9 +19,12 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser.query; +import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.tsfile.common.constant.TsFileConstant; @@ -41,6 +44,8 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -64,13 +69,20 @@ public class TsFileInsertionEventQueryParserTabletIterator implements Iterator currentModifications; + + // Maintain sorted mods list and current index for each measurement + private final Map, Integer>> measurementModsMap = new HashMap<>(); + TsFileInsertionEventQueryParserTabletIterator( final TsFileReader tsFileReader, final Map measurementDataTypeMap, final IDeviceID deviceId, final List measurements, final IExpression timeFilterExpression, - final PipeMemoryBlock allocatedBlockForTablet) + final PipeMemoryBlock allocatedBlockForTablet, + final PatternTreeMap currentModifications) throws IOException { this.tsFileReader = tsFileReader; this.measurementDataTypeMap = measurementDataTypeMap; @@ -90,6 +102,10 @@ public class TsFileInsertionEventQueryParserTabletIterator implements Iterator mods = currentModifications.getOverlapped(deviceId, measurement); + if (mods == null || mods.isEmpty()) { + // No mods, use empty list and index 0 + measurementModsMap.put(measurement, new Pair<>(Collections.EMPTY_LIST, 0)); + continue; + } + + // Sort by time range for efficient lookup + final List sortedMods = + mods.stream() + .filter(modification -> (!deviceId.isTableModel() || modification.affects(deviceId))) + .sorted( + (m1, m2) -> Long.compare(m1.getTimeRange().getMin(), m2.getTimeRange().getMin())) + .collect(Collectors.toList()); + + // Store sorted mods and start index + measurementModsMap.put(measurement, new Pair<>(sortedMods, 0)); + } + } + + private boolean isDelete(String measurementID, long time) { + final Pair, Integer> modsPair = measurementModsMap.get(measurementID); + if (modsPair == null) { + return false; + } + + final List mods = modsPair.getLeft(); + if (mods == null || mods.isEmpty()) { + return false; + } + + Integer currentIndex = modsPair.getRight(); + if (currentIndex == null || currentIndex < 0) { + return false; + } + + // Search from current index + for (int i = currentIndex; i < mods.size(); i++) { + final ModEntry mod = mods.get(i); + final long modStartTime = mod.getTimeRange().getMin(); + final long modEndTime = mod.getTimeRange().getMax(); + + if (time < modStartTime) { + // Current time is before mod start time, update index and return false + measurementModsMap.put(measurementID, new Pair<>(mods, i)); + return false; + } else if (time <= modEndTime) { + // Current time is within mod time range, update index and return true + measurementModsMap.put(measurementID, new Pair<>(mods, i)); + return true; + } + // If time > modEndTime, continue to next mod + } + + // All mods checked, clear mods list and reset index to 0 + measurementModsMap.put(measurementID, new Pair<>(Collections.EMPTY_LIST, 0)); + return false; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java index 47aba940a725..037aa2fc7df8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser.scan; +import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; @@ -28,6 +29,9 @@ import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; @@ -37,13 +41,16 @@ import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.reader.IChunkReader; import org.apache.tsfile.read.reader.chunk.AlignedChunkReader; import org.apache.tsfile.read.reader.chunk.ChunkReader; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.DateUtils; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.TsPrimitiveType; @@ -55,12 +62,14 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; +import java.util.stream.Collectors; public class TsFileInsertionEventScanParser extends TsFileInsertionEventParser { @@ -77,6 +86,8 @@ public class TsFileInsertionEventScanParser extends TsFileInsertionEventParser { private IDeviceID currentDevice; private boolean currentIsAligned; private final List currentMeasurements = new ArrayList<>(); + private final List> measurementTimeRangeList = new ArrayList<>(); + private final List measurementTimeRangeIndexList = new ArrayList<>(); // Cached time chunk private final List timeChunkList = new ArrayList<>(); @@ -88,6 +99,10 @@ public class TsFileInsertionEventScanParser extends TsFileInsertionEventParser { private byte lastMarker = Byte.MIN_VALUE; + // mods entry + private PatternTreeMap currentModifications = + PatternTreeMapFactory.getModsPatternTreeMap(); + public TsFileInsertionEventScanParser( final String pipeName, final long creationTime, @@ -114,6 +129,11 @@ public TsFileInsertionEventScanParser( PipeConfig.getInstance().getPipeMaxAlignedSeriesChunkSizeInOneBatch()); try { + ModificationFile.readAllModifications(tsFile, true) + .forEach( + modification -> + currentModifications.append(modification.keyOfPatternTree(), modification)); + tsFileSequenceReader = new TsFileSequenceReader(tsFile.getAbsolutePath(), false, false); tsFileSequenceReader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1); @@ -320,7 +340,14 @@ private void putValueToColumns(final BatchData data, final Tablet tablet, final if (data.getDataType() == TSDataType.VECTOR) { for (int i = 0; i < tablet.getSchemas().size(); ++i) { final TsPrimitiveType primitiveType = data.getVector()[i]; - if (Objects.isNull(primitiveType)) { + if (Objects.isNull(primitiveType) || isDelete(i, data.currentTime())) { + switch (tablet.getSchemas().get(i).getType()) { + case TEXT: + case BLOB: + case STRING: + tablet.addValue(rowIndex, i, Binary.EMPTY_VALUE.getValues()); + } + tablet.getBitMaps()[i].mark(rowIndex); continue; } switch (tablet.getSchemas().get(i).getType()) { @@ -389,6 +416,8 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { long valueChunkSize = 0; final List valueChunkList = new ArrayList<>(); currentMeasurements.clear(); + measurementTimeRangeList.clear(); + measurementTimeRangeIndexList.clear(); if (lastMarker == MetaMarker.SEPARATOR) { chunkReader = null; @@ -404,131 +433,180 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { case MetaMarker.TIME_CHUNK_HEADER: case MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER: case MetaMarker.ONLY_ONE_PAGE_TIME_CHUNK_HEADER: - // Notice that the data in one chunk group is either aligned or non-aligned - // There is no need to consider non-aligned chunks when there are value chunks - currentIsMultiPage = marker == MetaMarker.CHUNK_HEADER; + { + // Notice that the data in one chunk group is either aligned or non-aligned + // There is no need to consider non-aligned chunks when there are value chunks + currentIsMultiPage = marker == MetaMarker.CHUNK_HEADER; - chunkHeader = tsFileSequenceReader.readChunkHeader(marker); - - if (Objects.isNull(currentDevice)) { - tsFileSequenceReader.position( - tsFileSequenceReader.position() + chunkHeader.getDataSize()); - break; - } - - if ((chunkHeader.getChunkType() & TsFileConstant.TIME_COLUMN_MASK) - == TsFileConstant.TIME_COLUMN_MASK) { - timeChunkList.add( - new Chunk( - chunkHeader, tsFileSequenceReader.readChunk(-1, chunkHeader.getDataSize()))); - isMultiPageList.add(marker == MetaMarker.TIME_CHUNK_HEADER); - break; - } + chunkHeader = tsFileSequenceReader.readChunkHeader(marker); - if (!treePattern.matchesMeasurement(currentDevice, chunkHeader.getMeasurementID())) { - tsFileSequenceReader.position( - tsFileSequenceReader.position() + chunkHeader.getDataSize()); - break; - } + final long nextChunkHeaderOffset = + tsFileSequenceReader.position() + chunkHeader.getDataSize(); - if (chunkHeader.getDataSize() > allocatedMemoryBlockForChunk.getMemoryUsageInBytes()) { - PipeDataNodeResourceManager.memory() - .forceResize(allocatedMemoryBlockForChunk, chunkHeader.getDataSize()); - } + if (Objects.isNull(currentDevice)) { + tsFileSequenceReader.position(nextChunkHeaderOffset); + break; + } - chunkReader = - currentIsMultiPage - ? new ChunkReader( - new Chunk( - chunkHeader, - tsFileSequenceReader.readChunk(-1, chunkHeader.getDataSize())), - filter) - : new SinglePageWholeChunkReader( - new Chunk( - chunkHeader, - tsFileSequenceReader.readChunk(-1, chunkHeader.getDataSize()))); - currentIsAligned = false; - currentMeasurements.add( - new MeasurementSchema(chunkHeader.getMeasurementID(), chunkHeader.getDataType())); - return; - case MetaMarker.VALUE_CHUNK_HEADER: - case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER: - if (Objects.isNull(firstChunkHeader4NextSequentialValueChunks)) { - chunkHeader = tsFileSequenceReader.readChunkHeader(marker); + if ((chunkHeader.getChunkType() & TsFileConstant.TIME_COLUMN_MASK) + == TsFileConstant.TIME_COLUMN_MASK) { + timeChunkList.add( + new Chunk( + chunkHeader, tsFileSequenceReader.readChunk(-1, chunkHeader.getDataSize()))); + isMultiPageList.add(marker == MetaMarker.TIME_CHUNK_HEADER); + break; + } - if (Objects.isNull(currentDevice) - || !treePattern.matchesMeasurement(currentDevice, chunkHeader.getMeasurementID())) { - tsFileSequenceReader.position( - tsFileSequenceReader.position() + chunkHeader.getDataSize()); + if (!treePattern.matchesMeasurement(currentDevice, chunkHeader.getMeasurementID())) { + tsFileSequenceReader.position(nextChunkHeaderOffset); break; } - // Increase value index - final int valueIndex = - measurementIndexMap.compute( - chunkHeader.getMeasurementID(), - (measurement, index) -> Objects.nonNull(index) ? index + 1 : 0); + if (chunkHeader.getDataSize() > allocatedMemoryBlockForChunk.getMemoryUsageInBytes()) { + PipeDataNodeResourceManager.memory() + .forceResize(allocatedMemoryBlockForChunk, chunkHeader.getDataSize()); + } - // Emit when encountered non-sequential value chunk, or the chunk size exceeds - // certain value to avoid OOM - // Do not record or end current value chunks when there are empty chunks - if (chunkHeader.getDataSize() == 0) { + Chunk chunk = + new Chunk( + chunkHeader, tsFileSequenceReader.readChunk(-1, chunkHeader.getDataSize())); + // Skip the chunk if it is fully deleted by mods + final Statistics statistics = chunk.getChunkStatistic(); + if (statistics != null + && isAllDeletedByMods( + currentDevice, + chunkHeader.getMeasurementID(), + statistics.getStartTime(), + statistics.getEndTime())) { break; } - boolean needReturn = false; - final long timeChunkSize = - lastIndex >= 0 - ? PipeMemoryWeightUtil.calculateChunkRamBytesUsed(timeChunkList.get(lastIndex)) - : 0; - if (lastIndex >= 0) { - if (valueIndex != lastIndex) { - needReturn = recordAlignedChunk(valueChunkList, marker); - } else { - final long chunkSize = timeChunkSize + valueChunkSize; - if (chunkSize + chunkHeader.getDataSize() - > allocatedMemoryBlockForChunk.getMemoryUsageInBytes()) { - if (valueChunkList.size() == 1 - && chunkSize > allocatedMemoryBlockForChunk.getMemoryUsageInBytes()) { - PipeDataNodeResourceManager.memory() - .forceResize(allocatedMemoryBlockForChunk, chunkSize); - } + + chunkReader = + currentIsMultiPage + ? new ChunkReader(chunk, filter) + : new SinglePageWholeChunkReader(chunk); + currentIsAligned = false; + currentMeasurements.add( + new MeasurementSchema(chunkHeader.getMeasurementID(), chunkHeader.getDataType())); + List timeRange = + getModTimeRanges(currentDevice, chunkHeader.getMeasurementID()); + if (timeRange.isEmpty()) { + measurementTimeRangeList.add(Collections.EMPTY_LIST); + measurementTimeRangeIndexList.add(-1); + } else { + measurementTimeRangeList.add(timeRange); + measurementTimeRangeIndexList.add(0); + } + return; + } + case MetaMarker.VALUE_CHUNK_HEADER: + case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER: + { + if (Objects.isNull(firstChunkHeader4NextSequentialValueChunks)) { + chunkHeader = tsFileSequenceReader.readChunkHeader(marker); + + final long nextChunkHeaderOffset = + tsFileSequenceReader.position() + chunkHeader.getDataSize(); + if (Objects.isNull(currentDevice) + || !treePattern.matchesMeasurement( + currentDevice, chunkHeader.getMeasurementID())) { + tsFileSequenceReader.position(nextChunkHeaderOffset); + break; + } + + // Increase value index + final int valueIndex = + measurementIndexMap.compute( + chunkHeader.getMeasurementID(), + (measurement, index) -> Objects.nonNull(index) ? index + 1 : 0); + + // Emit when encountered non-sequential value chunk, or the chunk size exceeds + // certain value to avoid OOM + // Do not record or end current value chunks when there are empty chunks + if (chunkHeader.getDataSize() == 0) { + break; + } + boolean needReturn = false; + final long timeChunkSize = + lastIndex >= 0 + ? PipeMemoryWeightUtil.calculateChunkRamBytesUsed( + timeChunkList.get(lastIndex)) + : 0; + if (lastIndex >= 0) { + if (valueIndex != lastIndex) { needReturn = recordAlignedChunk(valueChunkList, marker); + } else { + final long chunkSize = timeChunkSize + valueChunkSize; + if (chunkSize + chunkHeader.getDataSize() + > allocatedMemoryBlockForChunk.getMemoryUsageInBytes()) { + if (valueChunkList.size() == 1 + && chunkSize > allocatedMemoryBlockForChunk.getMemoryUsageInBytes()) { + PipeDataNodeResourceManager.memory() + .forceResize(allocatedMemoryBlockForChunk, chunkSize); + } + needReturn = recordAlignedChunk(valueChunkList, marker); + } } } + lastIndex = valueIndex; + if (needReturn) { + firstChunkHeader4NextSequentialValueChunks = chunkHeader; + return; + } + } else { + chunkHeader = firstChunkHeader4NextSequentialValueChunks; + firstChunkHeader4NextSequentialValueChunks = null; } - lastIndex = valueIndex; - if (needReturn) { - firstChunkHeader4NextSequentialValueChunks = chunkHeader; - return; + + Chunk chunk = + new Chunk( + chunkHeader, tsFileSequenceReader.readChunk(-1, chunkHeader.getDataSize())); + // Skip the chunk if it is fully deleted by mods + final Statistics statistics = chunk.getChunkStatistic(); + if (statistics != null + && isAllDeletedByMods( + currentDevice, + chunkHeader.getMeasurementID(), + statistics.getStartTime(), + statistics.getEndTime())) { + break; } - } else { - chunkHeader = firstChunkHeader4NextSequentialValueChunks; - firstChunkHeader4NextSequentialValueChunks = null; - } - valueChunkSize += chunkHeader.getDataSize(); - valueChunkList.add( - new Chunk( - chunkHeader, tsFileSequenceReader.readChunk(-1, chunkHeader.getDataSize()))); - currentMeasurements.add( - new MeasurementSchema(chunkHeader.getMeasurementID(), chunkHeader.getDataType())); - break; + valueChunkSize += chunkHeader.getDataSize(); + valueChunkList.add(chunk); + currentMeasurements.add( + new MeasurementSchema(chunkHeader.getMeasurementID(), chunkHeader.getDataType())); + List timeRange = + getModTimeRanges(currentDevice, chunkHeader.getMeasurementID()); + if (timeRange.isEmpty()) { + measurementTimeRangeList.add(Collections.EMPTY_LIST); + measurementTimeRangeIndexList.add(-1); + } else { + measurementTimeRangeList.add(timeRange); + measurementTimeRangeIndexList.add(0); + } + break; + } case MetaMarker.CHUNK_GROUP_HEADER: - // Return before "currentDevice" changes - if (recordAlignedChunk(valueChunkList, marker)) { - return; + { + // Return before "currentDevice" changes + if (recordAlignedChunk(valueChunkList, marker)) { + return; + } + // Clear because the cached data will never be used in the next chunk group + lastIndex = -1; + timeChunkList.clear(); + isMultiPageList.clear(); + measurementIndexMap.clear(); + final IDeviceID deviceID = tsFileSequenceReader.readChunkGroupHeader().getDeviceID(); + currentDevice = treePattern.mayOverlapWithDevice(deviceID) ? deviceID : null; + break; } - // Clear because the cached data will never be used in the next chunk group - lastIndex = -1; - timeChunkList.clear(); - isMultiPageList.clear(); - measurementIndexMap.clear(); - final IDeviceID deviceID = tsFileSequenceReader.readChunkGroupHeader().getDeviceID(); - currentDevice = treePattern.mayOverlapWithDevice(deviceID) ? deviceID : null; - break; case MetaMarker.OPERATION_INDEX_RANGE: - tsFileSequenceReader.readPlanIndex(); - break; + { + tsFileSequenceReader.readPlanIndex(); + break; + } default: MetaMarker.handleUnexpectedMarker(marker); } @@ -569,4 +647,45 @@ public void close() { allocatedMemoryBlockForChunk.close(); } } + + public boolean isAllDeletedByMods( + IDeviceID deviceID, String measurementID, long startTime, long endTime) { + final List mods = currentModifications.getOverlapped(deviceID, measurementID); + return mods.stream() + .anyMatch( + modification -> + modification.getTimeRange().contains(startTime, endTime) + && (!deviceID.isTableModel() || modification.affects(deviceID))); + } + + public List getModTimeRanges(IDeviceID deviceID, String measurementID) { + final List mods = currentModifications.getOverlapped(deviceID, measurementID); + return mods.stream() + .filter(modification -> (!deviceID.isTableModel() || modification.affects(deviceID))) + .map(ModEntry::getTimeRange) + .sorted() + .collect(Collectors.toList()); + } + + public boolean isDelete(int measurementIndex, long time) { + int index = measurementTimeRangeIndexList.get(measurementIndex); + if (index == -1 || measurementTimeRangeList.isEmpty()) { + return false; + } + + List timeRanges = measurementTimeRangeList.get(measurementIndex); + for (int i = index; i < timeRanges.size(); i++) { + TimeRange timeRange = timeRanges.get(i); + if (time < timeRange.getMin()) { + measurementTimeRangeIndexList.set(measurementIndex, i); + return false; + } else if (time >= timeRange.getMin() && time <= timeRange.getMax()) { + measurementTimeRangeIndexList.set(measurementIndex, i); + return true; + } + } + + measurementTimeRangeIndexList.set(measurementIndex, -1); + return false; + } } From 14ff393ac7c91bec52c9f83b1be25c00b0d8aa7f Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Thu, 9 Oct 2025 11:26:04 +0800 Subject: [PATCH 02/31] update --- .../query/TsFileInsertionEventQueryParserTabletIterator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java index 5024414b0399..76a14a17c2a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java @@ -104,7 +104,6 @@ public class TsFileInsertionEventQueryParserTabletIterator implements Iterator Date: Thu, 9 Oct 2025 16:07:55 +0800 Subject: [PATCH 03/31] add ModsOperationUtil --- .../TsFileInsertionEventQueryParser.java | 34 +-- ...sertionEventQueryParserTabletIterator.java | 73 +---- .../scan/TsFileInsertionEventScanParser.java | 100 ++----- ...sertionEventTableParserTabletIterator.java | 36 ++- .../tsfile/parser/util/ModsOperationUtil.java | 265 ++++++++++++++++++ 5 files changed, 334 insertions(+), 174 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java index c0128381b5ed..f2c479d25c91 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java @@ -27,12 +27,12 @@ import org.apache.iotdb.db.pipe.event.common.PipeInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.parser.TsFileInsertionEventParser; +import org.apache.iotdb.db.pipe.event.common.tsfile.parser.util.ModsOperationUtil; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; -import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; @@ -43,7 +43,6 @@ import org.apache.tsfile.read.TsFileDeviceIterator; import org.apache.tsfile.read.TsFileReader; import org.apache.tsfile.read.TsFileSequenceReader; -import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.record.Tablet; import org.slf4j.Logger; @@ -60,7 +59,6 @@ import java.util.NoSuchElementException; import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; public class TsFileInsertionEventQueryParser extends TsFileInsertionEventParser { @@ -125,11 +123,7 @@ public TsFileInsertionEventQueryParser( super(pipeName, creationTime, pattern, null, startTime, endTime, pipeTaskMeta, sourceEvent); try { - // Read mods file first - ModificationFile.readAllModifications(tsFile, true) - .forEach( - modification -> - currentModifications.append(modification.keyOfPatternTree(), modification)); + currentModifications = ModsOperationUtil.loadModificationsFromTsFile(tsFile); final PipeTsFileResourceManager tsFileResourceManager = PipeDataNodeResourceManager.tsfile(); final Map> deviceMeasurementsMap; @@ -208,11 +202,12 @@ public TsFileInsertionEventQueryParser( try { TimeseriesMetadata meta = tsFileSequenceReader.readTimeseriesMetadata(deviceId, measurement, true); - return isAllDeletedByMods( + return ModsOperationUtil.isAllDeletedByMods( deviceId, measurement, meta.getStatistics().getStartTime(), - meta.getStatistics().getEndTime()); + meta.getStatistics().getEndTime(), + currentModifications); } catch (IOException e) { LOGGER.warn( "Failed to read metadata for deviceId: {}, measurement: {}, removing", @@ -480,23 +475,4 @@ public void close() { allocatedMemoryBlock.close(); } } - - public boolean isAllDeletedByMods( - IDeviceID deviceID, String measurementID, long startTime, long endTime) { - final List mods = currentModifications.getOverlapped(deviceID, measurementID); - return mods.stream() - .anyMatch( - modification -> - modification.getTimeRange().contains(startTime, endTime) - && (!deviceID.isTableModel() || modification.affects(deviceID))); - } - - public List getModTimeRanges(IDeviceID deviceID, String measurementID) { - final List mods = currentModifications.getOverlapped(deviceID, measurementID); - return mods.stream() - .filter(modification -> (!deviceID.isTableModel() || modification.affects(deviceID))) - .map(ModEntry::getTimeRange) - .sorted() - .collect(Collectors.toList()); - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java index 76a14a17c2a1..16f09b5b5e58 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser.query; import org.apache.iotdb.commons.path.PatternTreeMap; +import org.apache.iotdb.db.pipe.event.common.tsfile.parser.util.ModsOperationUtil; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; @@ -44,8 +45,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -73,7 +72,7 @@ public class TsFileInsertionEventQueryParserTabletIterator implements Iterator currentModifications; // Maintain sorted mods list and current index for each measurement - private final Map, Integer>> measurementModsMap = new HashMap<>(); + private final List measurementModsList = new ArrayList<>(); TsFileInsertionEventQueryParserTabletIterator( final TsFileReader tsFileReader, @@ -104,7 +103,9 @@ public class TsFileInsertionEventQueryParserTabletIterator implements Iterator mods = currentModifications.getOverlapped(deviceId, measurement); - if (mods == null || mods.isEmpty()) { - // No mods, use empty list and index 0 - measurementModsMap.put(measurement, new Pair<>(Collections.EMPTY_LIST, 0)); - continue; - } - - // Sort by time range for efficient lookup - final List sortedMods = - mods.stream() - .filter(modification -> (!deviceId.isTableModel() || modification.affects(deviceId))) - .sorted( - (m1, m2) -> Long.compare(m1.getTimeRange().getMin(), m2.getTimeRange().getMin())) - .collect(Collectors.toList()); - - // Store sorted mods and start index - measurementModsMap.put(measurement, new Pair<>(sortedMods, 0)); - } - } - - private boolean isDelete(String measurementID, long time) { - final Pair, Integer> modsPair = measurementModsMap.get(measurementID); - if (modsPair == null) { - return false; - } - - final List mods = modsPair.getLeft(); - if (mods == null || mods.isEmpty()) { - return false; - } - - Integer currentIndex = modsPair.getRight(); - if (currentIndex == null || currentIndex < 0) { - return false; - } - - // Search from current index - for (int i = currentIndex; i < mods.size(); i++) { - final ModEntry mod = mods.get(i); - final long modStartTime = mod.getTimeRange().getMin(); - final long modEndTime = mod.getTimeRange().getMax(); - - if (time < modStartTime) { - // Current time is before mod start time, update index and return false - measurementModsMap.put(measurementID, new Pair<>(mods, i)); - return false; - } else if (time <= modEndTime) { - // Current time is within mod time range, update index and return true - measurementModsMap.put(measurementID, new Pair<>(mods, i)); - return true; - } - // If time > modEndTime, continue to next mod - } - - // All mods checked, clear mods list and reset index to 0 - measurementModsMap.put(measurementID, new Pair<>(Collections.EMPTY_LIST, 0)); - return false; - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java index 037aa2fc7df8..194b5eaa44b5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java @@ -26,11 +26,11 @@ import org.apache.iotdb.db.pipe.event.common.PipeInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.parser.TsFileInsertionEventParser; +import org.apache.iotdb.db.pipe.event.common.tsfile.parser.util.ModsOperationUtil; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; -import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; @@ -45,7 +45,6 @@ import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.Chunk; -import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.reader.IChunkReader; import org.apache.tsfile.read.reader.chunk.AlignedChunkReader; @@ -69,7 +68,6 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; -import java.util.stream.Collectors; public class TsFileInsertionEventScanParser extends TsFileInsertionEventParser { @@ -86,9 +84,7 @@ public class TsFileInsertionEventScanParser extends TsFileInsertionEventParser { private IDeviceID currentDevice; private boolean currentIsAligned; private final List currentMeasurements = new ArrayList<>(); - private final List> measurementTimeRangeList = new ArrayList<>(); - private final List measurementTimeRangeIndexList = new ArrayList<>(); - + private final List modsInfos = new ArrayList<>(); // Cached time chunk private final List timeChunkList = new ArrayList<>(); private final List isMultiPageList = new ArrayList<>(); @@ -100,8 +96,7 @@ public class TsFileInsertionEventScanParser extends TsFileInsertionEventParser { private byte lastMarker = Byte.MIN_VALUE; // mods entry - private PatternTreeMap currentModifications = - PatternTreeMapFactory.getModsPatternTreeMap(); + private PatternTreeMap currentModifications; public TsFileInsertionEventScanParser( final String pipeName, @@ -129,10 +124,7 @@ public TsFileInsertionEventScanParser( PipeConfig.getInstance().getPipeMaxAlignedSeriesChunkSizeInOneBatch()); try { - ModificationFile.readAllModifications(tsFile, true) - .forEach( - modification -> - currentModifications.append(modification.keyOfPatternTree(), modification)); + currentModifications = ModsOperationUtil.loadModificationsFromTsFile(tsFile); tsFileSequenceReader = new TsFileSequenceReader(tsFile.getAbsolutePath(), false, false); tsFileSequenceReader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1); @@ -340,7 +332,7 @@ private void putValueToColumns(final BatchData data, final Tablet tablet, final if (data.getDataType() == TSDataType.VECTOR) { for (int i = 0; i < tablet.getSchemas().size(); ++i) { final TsPrimitiveType primitiveType = data.getVector()[i]; - if (Objects.isNull(primitiveType) || isDelete(i, data.currentTime())) { + if (Objects.isNull(primitiveType) || ModsOperationUtil.isDelete(i, modsInfos.get(i))) { switch (tablet.getSchemas().get(i).getType()) { case TEXT: case BLOB: @@ -416,8 +408,7 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { long valueChunkSize = 0; final List valueChunkList = new ArrayList<>(); currentMeasurements.clear(); - measurementTimeRangeList.clear(); - measurementTimeRangeIndexList.clear(); + modsInfos.clear(); if (lastMarker == MetaMarker.SEPARATOR) { chunkReader = null; @@ -473,11 +464,12 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { // Skip the chunk if it is fully deleted by mods final Statistics statistics = chunk.getChunkStatistic(); if (statistics != null - && isAllDeletedByMods( + && ModsOperationUtil.isAllDeletedByMods( currentDevice, chunkHeader.getMeasurementID(), statistics.getStartTime(), - statistics.getEndTime())) { + statistics.getEndTime(), + currentModifications)) { break; } @@ -488,15 +480,11 @@ && isAllDeletedByMods( currentIsAligned = false; currentMeasurements.add( new MeasurementSchema(chunkHeader.getMeasurementID(), chunkHeader.getDataType())); - List timeRange = - getModTimeRanges(currentDevice, chunkHeader.getMeasurementID()); - if (timeRange.isEmpty()) { - measurementTimeRangeList.add(Collections.EMPTY_LIST); - measurementTimeRangeIndexList.add(-1); - } else { - measurementTimeRangeList.add(timeRange); - measurementTimeRangeIndexList.add(0); - } + modsInfos.addAll( + ModsOperationUtil.initializeMeasurementMods( + currentDevice, + Collections.singletonList(chunkHeader.getMeasurementID()), + currentModifications)); return; } case MetaMarker.VALUE_CHUNK_HEADER: @@ -564,11 +552,12 @@ && isAllDeletedByMods( // Skip the chunk if it is fully deleted by mods final Statistics statistics = chunk.getChunkStatistic(); if (statistics != null - && isAllDeletedByMods( + && ModsOperationUtil.isAllDeletedByMods( currentDevice, chunkHeader.getMeasurementID(), statistics.getStartTime(), - statistics.getEndTime())) { + statistics.getEndTime(), + currentModifications)) { break; } @@ -576,15 +565,11 @@ && isAllDeletedByMods( valueChunkList.add(chunk); currentMeasurements.add( new MeasurementSchema(chunkHeader.getMeasurementID(), chunkHeader.getDataType())); - List timeRange = - getModTimeRanges(currentDevice, chunkHeader.getMeasurementID()); - if (timeRange.isEmpty()) { - measurementTimeRangeList.add(Collections.EMPTY_LIST); - measurementTimeRangeIndexList.add(-1); - } else { - measurementTimeRangeList.add(timeRange); - measurementTimeRangeIndexList.add(0); - } + modsInfos.addAll( + ModsOperationUtil.initializeMeasurementMods( + currentDevice, + Collections.singletonList(chunkHeader.getMeasurementID()), + currentModifications)); break; } case MetaMarker.CHUNK_GROUP_HEADER: @@ -647,45 +632,4 @@ public void close() { allocatedMemoryBlockForChunk.close(); } } - - public boolean isAllDeletedByMods( - IDeviceID deviceID, String measurementID, long startTime, long endTime) { - final List mods = currentModifications.getOverlapped(deviceID, measurementID); - return mods.stream() - .anyMatch( - modification -> - modification.getTimeRange().contains(startTime, endTime) - && (!deviceID.isTableModel() || modification.affects(deviceID))); - } - - public List getModTimeRanges(IDeviceID deviceID, String measurementID) { - final List mods = currentModifications.getOverlapped(deviceID, measurementID); - return mods.stream() - .filter(modification -> (!deviceID.isTableModel() || modification.affects(deviceID))) - .map(ModEntry::getTimeRange) - .sorted() - .collect(Collectors.toList()); - } - - public boolean isDelete(int measurementIndex, long time) { - int index = measurementTimeRangeIndexList.get(measurementIndex); - if (index == -1 || measurementTimeRangeList.isEmpty()) { - return false; - } - - List timeRanges = measurementTimeRangeList.get(measurementIndex); - for (int i = index; i < timeRanges.size(); i++) { - TimeRange timeRange = timeRanges.get(i); - if (time < timeRange.getMin()) { - measurementTimeRangeIndexList.set(measurementIndex, i); - return false; - } else if (time >= timeRange.getMin() && time <= timeRange.getMax()) { - measurementTimeRangeIndexList.set(measurementIndex, i); - return true; - } - } - - measurementTimeRangeIndexList.set(measurementIndex, -1); - return false; - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java index 746d9d5b4a01..1a9c32c71e21 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java @@ -19,9 +19,13 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser.table; +import org.apache.iotdb.commons.path.PatternTreeMap; +import org.apache.iotdb.db.pipe.event.common.tsfile.parser.util.ModsOperationUtil; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.tsfile.enums.ColumnCategory; @@ -48,6 +52,7 @@ import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; +import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; @@ -75,6 +80,9 @@ public class TsFileInsertionEventTableParserTabletIterator implements Iterator modifications; + // Used to read tsfile data private IChunkReader chunkReader; private BatchData batchData; @@ -96,6 +104,8 @@ public class TsFileInsertionEventTableParserTabletIterator implements Iterator dataTypeList; private int deviceIdSize; + private List modsInfoList; + // Used to record whether the same Tablet is generated when parsing starts. Different table // information cannot be placed in the same Tablet. private boolean isSameTableName; @@ -112,6 +122,8 @@ public TsFileInsertionEventTableParserTabletIterator( final long startTime, final long endTime) throws IOException { + modifications = + ModsOperationUtil.loadModificationsFromTsFile(new File(tsFileSequenceReader.getFileName())); this.startTime = startTime; this.endTime = endTime; @@ -202,6 +214,25 @@ public boolean hasNext() { continue; } + Iterator iChunkMetadataIterator = + alignedChunkMetadata.getValueChunkMetadataList().iterator(); + if (iChunkMetadataIterator.hasNext()) { + IChunkMetadata iChunkMetadata = iChunkMetadataIterator.next(); + if (iChunkMetadata == null) { + throw new PipeException( + "Table model tsfile parsing does not support this type of ChunkMeta"); + } + + if (ModsOperationUtil.isAllDeletedByMods( + deviceID, + iChunkMetadata.getMeasurementUid(), + alignedChunkMetadata.getStartTime(), + alignedChunkMetadata.getEndTime(), + modifications)) { + iChunkMetadataIterator.remove(); + } + } + size += PipeMemoryWeightUtil.calculateAlignedChunkMetaBytesUsed(alignedChunkMetadata); if (allocatedMemoryBlockForChunkMeta.getMemoryUsageInBytes() < size) { @@ -386,6 +417,8 @@ private void initChunkReader(final AbstractAlignedChunkMetadata alignedChunkMeta } this.chunkReader = new TableChunkReader(timeChunk, valueChunkList, null); + this.modsInfoList = + ModsOperationUtil.initializeMeasurementMods(deviceID, measurementList, modifications); } private void fillMeasurementValueColumns( @@ -394,7 +427,8 @@ private void fillMeasurementValueColumns( for (int i = deviceIdSize, size = dataTypeList.size(); i < size; i++) { final TsPrimitiveType primitiveType = primitiveTypes[i - deviceIdSize]; - if (primitiveType == null) { + if (primitiveType == null + || ModsOperationUtil.isDelete(data.currentTime(), modsInfoList.get(i))) { switch (dataTypeList.get(i)) { case TEXT: case BLOB: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java new file mode 100644 index 000000000000..fed31db7a6cb --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.event.common.tsfile.parser.util; + +import org.apache.iotdb.commons.path.PatternTreeMap; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.utils.ModificationUtils; +import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; +import org.apache.iotdb.pipe.api.exception.PipeException; + +import org.apache.tsfile.file.metadata.IDeviceID; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Utility class for handling mods operations during TsFile parsing. Supports mods processing logic + * for both tree model and table model. + */ +public class ModsOperationUtil { + + private ModsOperationUtil() { + // Utility class, no instantiation allowed + } + + /** + * Load all modifications from TsFile and build PatternTreeMap + * + * @param tsFile TsFile file + * @return PatternTreeMap containing all modifications + */ + public static PatternTreeMap + loadModificationsFromTsFile(File tsFile) { + PatternTreeMap modifications = + PatternTreeMapFactory.getModsPatternTreeMap(); + + try { + ModificationFile.readAllModifications(tsFile, true) + .forEach( + modification -> modifications.append(modification.keyOfPatternTree(), modification)); + } catch (Exception e) { + throw new PipeException("Failed to load modifications from TsFile: " + tsFile.getPath(), e); + } + + return modifications; + } + + /** + * Check if data in the specified time range is completely deleted by mods Different logic for + * tree model and table model + * + * @param deviceID device ID + * @param measurementID measurement ID + * @param startTime start time + * @param endTime end time + * @param modifications modification records + * @return true if data is completely deleted, false otherwise + */ + public static boolean isAllDeletedByMods( + IDeviceID deviceID, + String measurementID, + long startTime, + long endTime, + PatternTreeMap modifications) { + if (modifications == null) { + return false; + } + + final List mods = modifications.getOverlapped(deviceID, measurementID); + if (mods == null || mods.isEmpty()) { + return false; + } + + // Different logic for tree model and table model + if (deviceID.isTableModel()) { + // For table model: check if any modification affects the device and covers the time range + return mods.stream() + .anyMatch( + modification -> + modification.getTimeRange().contains(startTime, endTime) + && modification.affects(deviceID) + && modification.affects(measurementID)); + } else { + // For tree model: check if any modification covers the time range + return mods.stream() + .anyMatch(modification -> modification.getTimeRange().contains(startTime, endTime)); + } + } + + /** + * Initialize mods mapping for specified measurement list + * + * @param deviceID device ID + * @param measurements measurement list + * @param modifications modification records + * @return mapping from measurement ID to mods list and index + */ + public static List initializeMeasurementMods( + IDeviceID deviceID, + List measurements, + PatternTreeMap modifications) { + + List modsInfos = new ArrayList<>(); + + for (final String measurement : measurements) { + final List mods = modifications.getOverlapped(deviceID, measurement); + if (mods == null || mods.isEmpty()) { + // No mods, use empty list and index 0 + modsInfos.add(new ModsInfo(Collections.emptyList(), 0)); + continue; + } + + // Sort by time range for efficient lookup + // Different filtering logic for tree model and table model + final List sortedMods; + if (deviceID.isTableModel()) { + // For table model: filter modifications that affect the device + sortedMods = + mods.stream() + .filter( + modification -> + modification.affects(deviceID) && modification.affects(measurement)) + .sorted() + .collect(Collectors.toList()); + } else { + // For tree model: no additional filtering needed + sortedMods = mods; + } + // Store sorted mods and start index + modsInfos.add(new ModsInfo(ModificationUtils.sortAndMerge(sortedMods), 0)); + } + + return modsInfos; + } + + /** + * Check if data at the specified time point is deleted + * + * @param time time point + * @param modsInfo mods information containing mods list and current index + * @return true if data is deleted, false otherwise + */ + public static boolean isDelete(long time, ModsInfo modsInfo) { + if (modsInfo == null) { + return false; + } + + final List mods = modsInfo.getMods(); + if (mods == null || mods.isEmpty()) { + return false; + } + + int currentIndex = modsInfo.getCurrentIndex(); + if (currentIndex < 0) { + return false; + } + + // Search from current index + for (int i = currentIndex; i < mods.size(); i++) { + final ModEntry mod = mods.get(i); + final long modStartTime = mod.getTimeRange().getMin(); + final long modEndTime = mod.getTimeRange().getMax(); + + if (time < modStartTime) { + // Current time is before mod start time, update index and return false + modsInfo.setCurrentIndex(i); + return false; + } else if (time <= modEndTime) { + // Current time is within mod time range, update index and return true + modsInfo.setCurrentIndex(i); + return true; + } + // If time > modEndTime, continue to next mod + } + + // All mods checked, clear mods list and reset index to 0 + modsInfo.setMods(Collections.emptyList()); + modsInfo.setCurrentIndex(0); + return false; + } + + /** + * Update index in measurement mods mapping + * + * @param measurementID measurement ID + * @param newIndex new index + * @param measurementModsMap measurement mods mapping + */ + public static void updateModsIndex( + String measurementID, int newIndex, Map measurementModsMap) { + final ModsInfo modsInfo = measurementModsMap.get(measurementID); + if (modsInfo != null) { + measurementModsMap.put(measurementID, new ModsInfo(modsInfo.getMods(), newIndex)); + } + } + + /** Mods information wrapper class, containing mods list and current index */ + public static class ModsInfo { + private List mods; + private int currentIndex; + + public ModsInfo(List mods, int currentIndex) { + this.mods = Objects.requireNonNull(mods); + this.currentIndex = currentIndex; + } + + public List getMods() { + return mods; + } + + public void setMods(List newMods) { + this.mods = newMods; + } + + public int getCurrentIndex() { + return currentIndex; + } + + public void setCurrentIndex(int newIndex) { + this.currentIndex = newIndex; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ModsInfo modsInfo = (ModsInfo) o; + return Objects.equals(mods, modsInfo.mods) + && Objects.equals(currentIndex, modsInfo.currentIndex); + } + + @Override + public int hashCode() { + return Objects.hash(mods, currentIndex); + } + + @Override + public String toString() { + return "ModsInfo{" + "mods=" + mods + ", currentIndex=" + currentIndex + '}'; + } + } +} From 1d4d0709986c338ab7511741aba5c97431203352 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Thu, 9 Oct 2025 19:00:20 +0800 Subject: [PATCH 04/31] update --- .../tsfile/PipeTsFileInsertionEvent.java | 2 +- .../TsFileInsertionEventParserProvider.java | 20 +++++++++----- .../TsFileInsertionEventQueryParser.java | 21 +++++++++------ ...sertionEventQueryParserTabletIterator.java | 14 ++++------ .../scan/TsFileInsertionEventScanParser.java | 15 +++++++---- .../TsFileInsertionEventTableParser.java | 26 ++++++++++++++++--- ...sertionEventTableParserTabletIterator.java | 6 ++--- ...tementDataTypeConvertExecutionVisitor.java | 3 ++- ...tementDataTypeConvertExecutionVisitor.java | 2 +- ...tementDataTypeConvertExecutionVisitor.java | 3 ++- ...tementDataTypeConvertExecutionVisitor.java | 8 +++++- .../event/TsFileInsertionEventParserTest.java | 2 +- 12 files changed, 80 insertions(+), 42 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java index 3617b7347ad9..2a1ab3f5a35f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java @@ -700,7 +700,7 @@ private TsFileInsertionEventParser initEventParser() { // To avoid renaming of the tsFile database shouldParse4Privilege ? userName : null, this) - .provide()); + .provide(isWithMod)); return eventParser.get(); } catch (final IOException e) { close(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java index a965c817b994..640d8cafd310 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserProvider.java @@ -78,7 +78,7 @@ public TsFileInsertionEventParserProvider( this.sourceEvent = sourceEvent; } - public TsFileInsertionEventParser provide() throws IOException { + public TsFileInsertionEventParser provide(final boolean isWithMod) throws IOException { if (pipeName != null) { PipeTsFileToTabletsMetrics.getInstance() .markTsFileToTabletInvocation(pipeName + "_" + creationTime); @@ -94,7 +94,8 @@ public TsFileInsertionEventParser provide() throws IOException { endTime, pipeTaskMeta, userName, - sourceEvent); + sourceEvent, + isWithMod); } // Use scan container to save memory @@ -109,7 +110,8 @@ public TsFileInsertionEventParser provide() throws IOException { startTime, endTime, pipeTaskMeta, - sourceEvent); + sourceEvent, + isWithMod); } if (treePattern instanceof IoTDBTreePattern @@ -128,7 +130,8 @@ public TsFileInsertionEventParser provide() throws IOException { startTime, endTime, pipeTaskMeta, - sourceEvent); + sourceEvent, + isWithMod); } final Map deviceIsAlignedMap = @@ -144,7 +147,8 @@ public TsFileInsertionEventParser provide() throws IOException { startTime, endTime, pipeTaskMeta, - sourceEvent); + sourceEvent, + isWithMod); } final int originalSize = deviceIsAlignedMap.size(); @@ -161,7 +165,8 @@ public TsFileInsertionEventParser provide() throws IOException { startTime, endTime, pipeTaskMeta, - sourceEvent) + sourceEvent, + isWithMod) : new TsFileInsertionEventQueryParser( pipeName, creationTime, @@ -171,7 +176,8 @@ public TsFileInsertionEventParser provide() throws IOException { endTime, pipeTaskMeta, sourceEvent, - filteredDeviceIsAlignedMap); + filteredDeviceIsAlignedMap, + isWithMod); } private Map filterDeviceIsAlignedMapByPattern( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java index f2c479d25c91..984d35601bee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java @@ -73,8 +73,7 @@ public class TsFileInsertionEventQueryParser extends TsFileInsertionEventParser private final Map measurementDataTypeMap; // mods entry - private PatternTreeMap currentModifications = - PatternTreeMapFactory.getModsPatternTreeMap(); + private PatternTreeMap currentModifications; @TestOnly public TsFileInsertionEventQueryParser( @@ -84,7 +83,7 @@ public TsFileInsertionEventQueryParser( final long endTime, final PipeInsertionEvent sourceEvent) throws IOException { - this(null, 0, tsFile, pattern, startTime, endTime, null, sourceEvent); + this(null, 0, tsFile, pattern, startTime, endTime, null, sourceEvent, false); } public TsFileInsertionEventQueryParser( @@ -95,7 +94,8 @@ public TsFileInsertionEventQueryParser( final long startTime, final long endTime, final PipeTaskMeta pipeTaskMeta, - final PipeInsertionEvent sourceEvent) + final PipeInsertionEvent sourceEvent, + final boolean isWithMod) throws IOException { this( pipeName, @@ -106,7 +106,8 @@ public TsFileInsertionEventQueryParser( endTime, pipeTaskMeta, sourceEvent, - null); + null, + isWithMod); } public TsFileInsertionEventQueryParser( @@ -118,12 +119,16 @@ public TsFileInsertionEventQueryParser( final long endTime, final PipeTaskMeta pipeTaskMeta, final PipeInsertionEvent sourceEvent, - final Map deviceIsAlignedMap) + final Map deviceIsAlignedMap, + final boolean isWithMod) throws IOException { super(pipeName, creationTime, pattern, null, startTime, endTime, pipeTaskMeta, sourceEvent); try { - currentModifications = ModsOperationUtil.loadModificationsFromTsFile(tsFile); + currentModifications = + isWithMod + ? ModsOperationUtil.loadModificationsFromTsFile(tsFile) + : PatternTreeMapFactory.getModsPatternTreeMap(); final PipeTsFileResourceManager tsFileResourceManager = PipeDataNodeResourceManager.tsfile(); final Map> deviceMeasurementsMap; @@ -171,7 +176,7 @@ public TsFileInsertionEventQueryParser( final Iterator>> iterator = deviceMeasurementsMap.entrySet().iterator(); - while (iterator.hasNext()) { + while (isWithMod && iterator.hasNext()) { final Map.Entry> entry = iterator.next(); final IDeviceID deviceId = entry.getKey(); final List measurements = entry.getValue(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java index 16f09b5b5e58..222fc33d1153 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java @@ -66,13 +66,10 @@ public class TsFileInsertionEventQueryParserTabletIterator implements Iterator currentModifications; - // Maintain sorted mods list and current index for each measurement - private final List measurementModsList = new ArrayList<>(); + private final List measurementModsList; + + private RowRecord rowRecord; TsFileInsertionEventQueryParserTabletIterator( final TsFileReader tsFileReader, @@ -101,11 +98,10 @@ public class TsFileInsertionEventQueryParserTabletIterator implements Iterator currentModifications; + private final PatternTreeMap currentModifications; public TsFileInsertionEventScanParser( final String pipeName, @@ -106,7 +106,8 @@ public TsFileInsertionEventScanParser( final long startTime, final long endTime, final PipeTaskMeta pipeTaskMeta, - final PipeInsertionEvent sourceEvent) + final PipeInsertionEvent sourceEvent, + final boolean isWithMod) throws IOException { super(pipeName, creationTime, pattern, null, startTime, endTime, pipeTaskMeta, sourceEvent); @@ -124,7 +125,10 @@ public TsFileInsertionEventScanParser( PipeConfig.getInstance().getPipeMaxAlignedSeriesChunkSizeInOneBatch()); try { - currentModifications = ModsOperationUtil.loadModificationsFromTsFile(tsFile); + currentModifications = + isWithMod + ? ModsOperationUtil.loadModificationsFromTsFile(tsFile) + : PatternTreeMapFactory.getModsPatternTreeMap(); tsFileSequenceReader = new TsFileSequenceReader(tsFile.getAbsolutePath(), false, false); tsFileSequenceReader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1); @@ -142,9 +146,10 @@ public TsFileInsertionEventScanParser( final long startTime, final long endTime, final PipeTaskMeta pipeTaskMeta, - final PipeInsertionEvent sourceEvent) + final PipeInsertionEvent sourceEvent, + final boolean isWithMod) throws IOException { - this(null, 0, tsFile, pattern, startTime, endTime, pipeTaskMeta, sourceEvent); + this(null, 0, tsFile, pattern, startTime, endTime, pipeTaskMeta, sourceEvent, isWithMod); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParser.java index 7a5d869aa1a0..93fc0cde81e7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParser.java @@ -28,9 +28,11 @@ import org.apache.iotdb.db.pipe.event.common.PipeInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.parser.TsFileInsertionEventParser; +import org.apache.iotdb.db.pipe.event.common.tsfile.parser.util.ModsOperationUtil; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; +import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; @@ -49,6 +51,7 @@ public class TsFileInsertionEventTableParser extends TsFileInsertionEventParser private final long endTime; private final TablePattern tablePattern; private final String userName; + private final boolean isWithMod; private final PipeMemoryBlock allocatedMemoryBlockForBatchData; private final PipeMemoryBlock allocatedMemoryBlockForChunk; @@ -64,10 +67,12 @@ public TsFileInsertionEventTableParser( final long endTime, final PipeTaskMeta pipeTaskMeta, final String userName, - final PipeInsertionEvent sourceEvent) + final PipeInsertionEvent sourceEvent, + final boolean isWithMod) throws IOException { super(pipeName, creationTime, null, pattern, startTime, endTime, pipeTaskMeta, sourceEvent); + this.isWithMod = isWithMod; try { long tableSize = Math.min( @@ -106,9 +111,20 @@ public TsFileInsertionEventTableParser( final long endTime, final PipeTaskMeta pipeTaskMeta, final String userName, - final PipeInsertionEvent sourceEvent) + final PipeInsertionEvent sourceEvent, + final boolean isWithMod) throws IOException { - this(null, 0, tsFile, pattern, startTime, endTime, pipeTaskMeta, userName, sourceEvent); + this( + null, + 0, + tsFile, + pattern, + startTime, + endTime, + pipeTaskMeta, + userName, + sourceEvent, + isWithMod); } @Override @@ -136,6 +152,10 @@ && hasTablePrivilege(entry.getKey()), allocatedMemoryBlockForChunk, allocatedMemoryBlockForChunkMeta, allocatedMemoryBlockForTableSchemas, + isWithMod + ? ModsOperationUtil.loadModificationsFromTsFile( + new File(tsFileSequenceReader.getFileName())) + : PatternTreeMapFactory.getModsPatternTreeMap(), startTime, endTime); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java index 1a9c32c71e21..39cd87b794d4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java @@ -52,7 +52,6 @@ import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; @@ -119,11 +118,10 @@ public TsFileInsertionEventTableParserTabletIterator( final PipeMemoryBlock allocatedMemoryBlockForChunk, final PipeMemoryBlock allocatedMemoryBlockForChunkMeta, final PipeMemoryBlock allocatedMemoryBlockForTableSchema, + final PatternTreeMap modifications, final long startTime, final long endTime) throws IOException { - modifications = - ModsOperationUtil.loadModificationsFromTsFile(new File(tsFileSequenceReader.getFileName())); this.startTime = startTime; this.endTime = endTime; @@ -216,7 +214,7 @@ public boolean hasNext() { Iterator iChunkMetadataIterator = alignedChunkMetadata.getValueChunkMetadataList().iterator(); - if (iChunkMetadataIterator.hasNext()) { + while (iChunkMetadataIterator.hasNext()) { IChunkMetadata iChunkMetadata = iChunkMetadataIterator.next(); if (iChunkMetadata == null) { throw new PipeException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java index 5fb87e550dfc..2ddcdd27f62d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTableStatementDataTypeConvertExecutionVisitor.java @@ -133,7 +133,8 @@ public Optional visitLoadFile( Long.MAX_VALUE, null, "root", - null)) { + null, + true)) { for (final TabletInsertionEvent tabletInsertionEvent : parser.toTabletInsertionEvents()) { if (!(tabletInsertionEvent instanceof PipeRawTabletInsertionEvent)) { continue; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTreeStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTreeStatementDataTypeConvertExecutionVisitor.java index 77bbf49094c3..8541e9bc9d02 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTreeStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeTreeStatementDataTypeConvertExecutionVisitor.java @@ -102,7 +102,7 @@ public Optional visitLoadFile( for (final File file : loadTsFileStatement.getTsFiles()) { try (final TsFileInsertionEventScanParser parser = new TsFileInsertionEventScanParser( - file, new IoTDBTreePattern(null), Long.MIN_VALUE, Long.MAX_VALUE, null, null)) { + file, new IoTDBTreePattern(null), Long.MIN_VALUE, Long.MAX_VALUE, null, null, true)) { for (final Pair tabletWithIsAligned : parser.toTabletWithIsAligneds()) { final PipeConvertedInsertTabletStatement statement = new PipeConvertedInsertTabletStatement( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java index 667a2faf0de7..eca63e40356e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java @@ -83,7 +83,8 @@ public Optional visitLoadTsFile( Long.MAX_VALUE, null, "root", - null)) { + null, + true)) { for (final TabletInsertionEvent tabletInsertionEvent : parser.toTabletInsertionEvents()) { if (!(tabletInsertionEvent instanceof PipeRawTabletInsertionEvent)) { continue; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java index 6dd851d1d9d9..d3ad0ae0a34b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java @@ -93,7 +93,13 @@ public Optional visitLoadFile( for (final File file : loadTsFileStatement.getTsFiles()) { try (final TsFileInsertionEventScanParser parser = new TsFileInsertionEventScanParser( - file, new IoTDBTreePattern(null), Long.MIN_VALUE, Long.MAX_VALUE, null, null)) { + file, + new IoTDBTreePattern(null), + Long.MIN_VALUE, + Long.MAX_VALUE, + null, + null, + true)) { for (final Pair tabletWithIsAligned : parser.toTabletWithIsAligneds()) { final PipeTransferTabletRawReq tabletRawReq = PipeTransferTabletRawReq.toTPipeTransferRawReq( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionEventParserTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionEventParserTest.java index 9c58ae9f0f68..7bfde3b158d5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionEventParserTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionEventParserTest.java @@ -584,7 +584,7 @@ private void testTsFilePointNum( ? new TsFileInsertionEventQueryParser( tsFile, pattern, startTime, endTime, tsFileInsertionEvent) : new TsFileInsertionEventScanParser( - tsFile, pattern, startTime, endTime, null, tsFileInsertionEvent)) { + tsFile, pattern, startTime, endTime, null, tsFileInsertionEvent, false)) { final AtomicInteger count1 = new AtomicInteger(0); final AtomicInteger count2 = new AtomicInteger(0); final AtomicInteger count3 = new AtomicInteger(0); From 6a9a93a3fa3d3ef1a5802cd3f1db571cd533bd2e Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Fri, 10 Oct 2025 10:33:37 +0800 Subject: [PATCH 05/31] update --- .../tsfile/parser/TsFileInsertionEventParser.java | 6 ++++++ .../query/TsFileInsertionEventQueryParser.java | 5 ----- .../scan/TsFileInsertionEventScanParser.java | 15 +++++---------- .../table/TsFileInsertionEventTableParser.java | 10 ++++++---- ...leInsertionEventTableParserTabletIterator.java | 1 + 5 files changed, 18 insertions(+), 19 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParser.java index 358103175fe7..7ecbb683ac16 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParser.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser; +import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; @@ -27,6 +28,8 @@ import org.apache.iotdb.db.pipe.metric.overview.PipeTsFileToTabletsMetrics; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.tsfile.read.TsFileSequenceReader; @@ -53,6 +56,9 @@ public abstract class TsFileInsertionEventParser implements AutoCloseable { protected final PipeTaskMeta pipeTaskMeta; // used to report progress protected final PipeInsertionEvent sourceEvent; // used to report progress + // mods entry + protected PatternTreeMap currentModifications; + protected final long initialTimeNano = System.nanoTime(); protected boolean timeUsageReported = false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java index 984d35601bee..1786561c841a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser.query; -import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; @@ -32,7 +31,6 @@ import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager; -import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; @@ -72,9 +70,6 @@ public class TsFileInsertionEventQueryParser extends TsFileInsertionEventParser private final Map deviceIsAlignedMap; private final Map measurementDataTypeMap; - // mods entry - private PatternTreeMap currentModifications; - @TestOnly public TsFileInsertionEventQueryParser( final File tsFile, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java index b3dc42381e0c..5bb4d645cf29 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.parser.scan; -import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern; @@ -30,7 +29,6 @@ import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; -import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; @@ -95,9 +93,6 @@ public class TsFileInsertionEventScanParser extends TsFileInsertionEventParser { private byte lastMarker = Byte.MIN_VALUE; - // mods entry - private final PatternTreeMap currentModifications; - public TsFileInsertionEventScanParser( final String pipeName, final long creationTime, @@ -436,11 +431,11 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { chunkHeader = tsFileSequenceReader.readChunkHeader(marker); - final long nextChunkHeaderOffset = + final long nextMarkerOffset = tsFileSequenceReader.position() + chunkHeader.getDataSize(); if (Objects.isNull(currentDevice)) { - tsFileSequenceReader.position(nextChunkHeaderOffset); + tsFileSequenceReader.position(nextMarkerOffset); break; } @@ -454,7 +449,7 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { } if (!treePattern.matchesMeasurement(currentDevice, chunkHeader.getMeasurementID())) { - tsFileSequenceReader.position(nextChunkHeaderOffset); + tsFileSequenceReader.position(nextMarkerOffset); break; } @@ -498,12 +493,12 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { if (Objects.isNull(firstChunkHeader4NextSequentialValueChunks)) { chunkHeader = tsFileSequenceReader.readChunkHeader(marker); - final long nextChunkHeaderOffset = + final long nextMarkerOffset = tsFileSequenceReader.position() + chunkHeader.getDataSize(); if (Objects.isNull(currentDevice) || !treePattern.matchesMeasurement( currentDevice, chunkHeader.getMeasurementID())) { - tsFileSequenceReader.position(nextChunkHeaderOffset); + tsFileSequenceReader.position(nextMarkerOffset); break; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParser.java index 93fc0cde81e7..05909c5ecf7a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParser.java @@ -74,6 +74,11 @@ public TsFileInsertionEventTableParser( this.isWithMod = isWithMod; try { + currentModifications = + this.isWithMod + ? ModsOperationUtil.loadModificationsFromTsFile( + new File(tsFileSequenceReader.getFileName())) + : PatternTreeMapFactory.getModsPatternTreeMap(); long tableSize = Math.min( PipeConfig.getInstance().getPipeDataStructureTabletSizeInBytes(), @@ -152,10 +157,7 @@ && hasTablePrivilege(entry.getKey()), allocatedMemoryBlockForChunk, allocatedMemoryBlockForChunkMeta, allocatedMemoryBlockForTableSchemas, - isWithMod - ? ModsOperationUtil.loadModificationsFromTsFile( - new File(tsFileSequenceReader.getFileName())) - : PatternTreeMapFactory.getModsPatternTreeMap(), + currentModifications, startTime, endTime); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java index 39cd87b794d4..aecd02548098 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java @@ -125,6 +125,7 @@ public TsFileInsertionEventTableParserTabletIterator( this.startTime = startTime; this.endTime = endTime; + this.modifications = modifications; this.reader = tsFileSequenceReader; this.metadataQuerier = new MetadataQuerierByFileImpl(reader); From 7c7742f2bb4cfe09bfdf7f27c736c37889721cf2 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Fri, 10 Oct 2025 10:50:30 +0800 Subject: [PATCH 06/31] add memory manage --- .../tsfile/parser/TsFileInsertionEventParser.java | 10 ++++++++++ .../parser/query/TsFileInsertionEventQueryParser.java | 3 +++ .../parser/scan/TsFileInsertionEventScanParser.java | 3 +++ .../parser/table/TsFileInsertionEventTableParser.java | 8 +++++--- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParser.java index 7ecbb683ac16..b68d9492426c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParser.java @@ -57,6 +57,7 @@ public abstract class TsFileInsertionEventParser implements AutoCloseable { protected final PipeInsertionEvent sourceEvent; // used to report progress // mods entry + protected PipeMemoryBlock allocatedMemoryBlockForModifications; protected PatternTreeMap currentModifications; protected final long initialTimeNano = System.nanoTime(); @@ -130,5 +131,14 @@ public void close() { if (allocatedMemoryBlockForTablet != null) { allocatedMemoryBlockForTablet.close(); } + + if (currentModifications != null) { + // help GC + currentModifications = null; + } + + if (allocatedMemoryBlockForModifications != null) { + allocatedMemoryBlockForModifications.close(); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java index 1786561c841a..db0e105dcb5e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java @@ -124,6 +124,9 @@ public TsFileInsertionEventQueryParser( isWithMod ? ModsOperationUtil.loadModificationsFromTsFile(tsFile) : PatternTreeMapFactory.getModsPatternTreeMap(); + allocatedMemoryBlockForModifications = + PipeDataNodeResourceManager.memory() + .forceAllocateForTabletWithRetry(currentModifications.ramBytesUsed()); final PipeTsFileResourceManager tsFileResourceManager = PipeDataNodeResourceManager.tsfile(); final Map> deviceMeasurementsMap; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java index 5bb4d645cf29..df0548f2fa79 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java @@ -124,6 +124,9 @@ public TsFileInsertionEventScanParser( isWithMod ? ModsOperationUtil.loadModificationsFromTsFile(tsFile) : PatternTreeMapFactory.getModsPatternTreeMap(); + allocatedMemoryBlockForModifications = + PipeDataNodeResourceManager.memory() + .forceAllocateForTabletWithRetry(currentModifications.ramBytesUsed()); tsFileSequenceReader = new TsFileSequenceReader(tsFile.getAbsolutePath(), false, false); tsFileSequenceReader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParser.java index 05909c5ecf7a..77a3a8916c9c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParser.java @@ -75,10 +75,12 @@ public TsFileInsertionEventTableParser( this.isWithMod = isWithMod; try { currentModifications = - this.isWithMod - ? ModsOperationUtil.loadModificationsFromTsFile( - new File(tsFileSequenceReader.getFileName())) + isWithMod + ? ModsOperationUtil.loadModificationsFromTsFile(tsFile) : PatternTreeMapFactory.getModsPatternTreeMap(); + allocatedMemoryBlockForModifications = + PipeDataNodeResourceManager.memory() + .forceAllocateForTabletWithRetry(currentModifications.ramBytesUsed()); long tableSize = Math.min( PipeConfig.getInstance().getPipeDataStructureTabletSizeInBytes(), From 1a093ed6ad1afb370425588b14505b8760b3f446 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Fri, 10 Oct 2025 11:57:33 +0800 Subject: [PATCH 07/31] fix --- ...sertionEventTableParserTabletIterator.java | 2 +- .../tsfile/parser/util/ModsOperationUtil.java | 19 +------------------ 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java index aecd02548098..1dae9aacd6e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java @@ -223,7 +223,7 @@ public boolean hasNext() { } if (ModsOperationUtil.isAllDeletedByMods( - deviceID, + pair.getLeft(), iChunkMetadata.getMeasurementUid(), alignedChunkMetadata.getStartTime(), alignedChunkMetadata.getEndTime(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java index fed31db7a6cb..795577cdfe45 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java @@ -32,7 +32,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; @@ -123,7 +122,7 @@ public static List initializeMeasurementMods( List measurements, PatternTreeMap modifications) { - List modsInfos = new ArrayList<>(); + List modsInfos = new ArrayList<>(measurements.size()); for (final String measurement : measurements) { final List mods = modifications.getOverlapped(deviceID, measurement); @@ -143,7 +142,6 @@ public static List initializeMeasurementMods( .filter( modification -> modification.affects(deviceID) && modification.affects(measurement)) - .sorted() .collect(Collectors.toList()); } else { // For tree model: no additional filtering needed @@ -202,21 +200,6 @@ public static boolean isDelete(long time, ModsInfo modsInfo) { return false; } - /** - * Update index in measurement mods mapping - * - * @param measurementID measurement ID - * @param newIndex new index - * @param measurementModsMap measurement mods mapping - */ - public static void updateModsIndex( - String measurementID, int newIndex, Map measurementModsMap) { - final ModsInfo modsInfo = measurementModsMap.get(measurementID); - if (modsInfo != null) { - measurementModsMap.put(measurementID, new ModsInfo(modsInfo.getMods(), newIndex)); - } - } - /** Mods information wrapper class, containing mods list and current index */ public static class ModsInfo { private List mods; From 938fa51c082036274475b59921610450c79aa488 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Fri, 10 Oct 2025 19:06:31 +0800 Subject: [PATCH 08/31] add table model IT --- ...oTDBPipeTsFileDecompositionWithModsIT.java | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java new file mode 100644 index 000000000000..9fdcb33f4eec --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual; + +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Collections; +import java.util.HashSet; + +import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQueryWithRetry; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeTsFileDecompositionWithModsIT extends AbstractPipeTableModelDualManualIT { + + @Test + public void testTsFileDecompositionWithMods() { + TableModelUtils.createDataBaseAndTable(senderEnv, "table1", "sg1"); + TableModelUtils.createDataBaseAndTable(receiverEnv, "table1", "sg1"); + + TableModelUtils.insertData("sg1", "table1", 1, 6, senderEnv); + + executeNonQueryWithRetry(senderEnv, "FLUSH"); + + executeNonQueryWithRetry( + senderEnv, "DELETE FROM table1 WHERE time >= 2 AND time <= 4",SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD,"sg1", "table"); + + executeNonQueryWithRetry( + senderEnv, "DELETE FROM table1 WHERE time >= 3 AND time <= 5",SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD,"sg1", "table"); + + executeNonQueryWithRetry(senderEnv, "FLUSH"); + + executeNonQueryWithRetry( + senderEnv, + String.format( + "CREATE PIPE test_pipe WITH SOURCE ('mods.enable'='true', 'capture.table'='true') WITH CONNECTOR('ip'='%s', 'port'='%s', 'username'='root', 'format'='tablet')", + receiverEnv.getDataNodeWrapperList().get(0).getIp(), + receiverEnv.getDataNodeWrapperList().get(0).getPort())); + + HashSet expectedResults = new HashSet<>(); + expectedResults.add("t1,t1,t1,t1,1,1.0,1,1970-01-01T00:00:00.001Z,1,1.0,1970-01-01,1,1970-01-01T00:00:00.001Z,"); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + TableModelUtils.getQuerySql("table1"), + TableModelUtils.generateHeaderResults(), + expectedResults, + "sg1"); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT s4 FROM table1 WHERE time >= 2 AND time <= 4", + "s4,", + Collections.emptySet(), + "sg1"); + } +} \ No newline at end of file From 4a801dd38766da57667db8c285f28e3323049a97 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Fri, 10 Oct 2025 19:20:27 +0800 Subject: [PATCH 09/31] add tree model IT --- ...oTDBPipeTsFileDecompositionWithModsIT.java | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java new file mode 100644 index 000000000000..18cfa694e22e --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.treemodel.manual; + +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Collections; +import java.util.HashSet; + +import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQueryWithRetry; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeTsFileDecompositionWithModsIT extends AbstractPipeDualTreeModelManualIT { + + @Test + public void testTsFileDecompositionWithMods() throws Exception { + TestUtils.executeNonQueryWithRetry(senderEnv, "CREATE DATABASE root.sg1"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "CREATE DATABASE root.sg1"); + + TestUtils.executeNonQueryWithRetry( + senderEnv, "CREATE ALIGNED TIMESERIES root.sg1.d1(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); + TestUtils.executeNonQueryWithRetry( + senderEnv, "CREATE TIMESERIES root.sg1.d2.s1 WITH DATATYPE=FLOAT"); + TestUtils.executeNonQueryWithRetry( + senderEnv, "CREATE TIMESERIES root.sg1.d2.s2 WITH DATATYPE=FLOAT"); + TestUtils.executeNonQueryWithRetry( + senderEnv, "CREATE ALIGNED TIMESERIES root.sg1.d3(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); + + TestUtils.executeNonQueryWithRetry( + receiverEnv, "CREATE ALIGNED TIMESERIES root.sg1.d1(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); + TestUtils.executeNonQueryWithRetry( + receiverEnv, "CREATE TIMESERIES root.sg1.d2.s1 WITH DATATYPE=FLOAT"); + TestUtils.executeNonQueryWithRetry( + receiverEnv, "CREATE TIMESERIES root.sg1.d2.s2 WITH DATATYPE=FLOAT"); + TestUtils.executeNonQueryWithRetry( + receiverEnv, "CREATE ALIGNED TIMESERIES root.sg1.d3(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); + + TestUtils.executeNonQueryWithRetry( + senderEnv, + "INSERT INTO root.sg1.d1(time, s1, s2, s3) ALIGNED VALUES (1, 1.0, 2.0, 3.0), (2, 1.1, 2.1, 3.1), (3, 1.2, 2.2, 3.2), (4, 1.3, 2.3, 3.3), (5, 1.4, 2.4, 3.4)"); + + TestUtils.executeNonQueryWithRetry( + senderEnv, + "INSERT INTO root.sg1.d2(time, s1, s2) VALUES (1, 10.0, 20.0), (2, 10.1, 20.1), (3, 10.2, 20.2), (4, 10.3, 20.3), (5, 10.4, 20.4)"); + + TestUtils.executeNonQueryWithRetry( + senderEnv, + "INSERT INTO root.sg1.d3(time, s1, s2, s3) ALIGNED VALUES (1, 100.0, 200.0, 300.0), (2, 100.1, 200.1, 300.1), (3, 100.2, 200.2, 300.2)"); + + TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + + TestUtils.executeNonQueryWithRetry( + senderEnv, "DELETE FROM root.sg1.d1.s1 WHERE time >= 2 AND time <= 4"); + + TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d2.*"); + + TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d3.*"); + + TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + + executeNonQueryWithRetry( + senderEnv, + String.format( + "CREATE PIPE test_pipe WITH SOURCE ('mods.enable'='true') WITH CONNECTOR('ip'='%s', 'port'='%s', 'username'='root', 'format'='tablet')", + receiverEnv.getDataNodeWrapperList().get(0).getIp(), + receiverEnv.getDataNodeWrapperList().get(0).getPort())); + + HashSet results = new HashSet<>(); + results.add("1,3.0,1.0,2.0,"); + results.add("2,3.1,null,2.1,"); + results.add("3,3.2,null,2.2,"); + results.add("4,3.3,null,2.3,"); + results.add("5,3.4,1.4,2.4,"); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT * FROM root.sg1.d1 ORDER BY time", + "Time,root.sg1.d1.s3,root.sg1.d1.s1,root.sg1.d1.s2,", + results); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "SELECT * FROM root.sg1.d2 ORDER BY time", "Time,root.sg1.d2.s1,root.sg1.d2.s2,", Collections.emptySet()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "SELECT * FROM root.sg1.d3 ORDER BY time", "Time,root.sg1.d3.s3,root.sg1.d3.s1,root.sg1.d3.s2,", Collections.emptySet()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "SELECT s1 FROM root.sg1.d1 WHERE time >= 2 AND time <= 4", "Time,root.sg1.d1.s1,", Collections.emptySet()); + } +} From 9a07c9d924f951f6e43f9518e2a6c7db2f50f3bb Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 14 Oct 2025 10:50:54 +0800 Subject: [PATCH 10/31] update --- ...oTDBPipeTsFileDecompositionWithModsIT.java | 18 +++- ...oTDBPipeTsFileDecompositionWithModsIT.java | 83 +++++++++++++++---- .../scan/TsFileInsertionEventScanParser.java | 55 ++++++++++-- 3 files changed, 130 insertions(+), 26 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index 9fdcb33f4eec..061197b25709 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -20,7 +20,6 @@ package org.apache.iotdb.pipe.it.dual.tablemodel.manual; import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; @@ -49,10 +48,20 @@ public void testTsFileDecompositionWithMods() { executeNonQueryWithRetry(senderEnv, "FLUSH"); executeNonQueryWithRetry( - senderEnv, "DELETE FROM table1 WHERE time >= 2 AND time <= 4",SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD,"sg1", "table"); + senderEnv, + "DELETE FROM table1 WHERE time >= 2 AND time <= 4", + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + "sg1", + "table"); executeNonQueryWithRetry( - senderEnv, "DELETE FROM table1 WHERE time >= 3 AND time <= 5",SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD,"sg1", "table"); + senderEnv, + "DELETE FROM table1 WHERE time >= 3 AND time <= 5", + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + "sg1", + "table"); executeNonQueryWithRetry(senderEnv, "FLUSH"); @@ -64,7 +73,8 @@ public void testTsFileDecompositionWithMods() { receiverEnv.getDataNodeWrapperList().get(0).getPort())); HashSet expectedResults = new HashSet<>(); - expectedResults.add("t1,t1,t1,t1,1,1.0,1,1970-01-01T00:00:00.001Z,1,1.0,1970-01-01,1,1970-01-01T00:00:00.001Z,"); + expectedResults.add( + "t1,t1,t1,t1,1,1.0,1,1970-01-01T00:00:00.001Z,1,1.0,1970-01-01,1,1970-01-01T00:00:00.001Z,"); TestUtils.assertDataEventuallyOnEnv( receiverEnv, diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index 18cfa694e22e..e998c63a9d69 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.pipe.it.dual.treemodel.manual; +import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; @@ -36,6 +37,15 @@ @Category({MultiClusterIT2DualTreeManual.class}) public class IoTDBPipeTsFileDecompositionWithModsIT extends AbstractPipeDualTreeModelManualIT { + @Override + protected void setupConfig() { + super.setupConfig(); + receiverEnv + .getConfig() + .getCommonConfig() + .setAutoCreateSchemaEnabled(true); + } + @Test public void testTsFileDecompositionWithMods() throws Exception { TestUtils.executeNonQueryWithRetry(senderEnv, "CREATE DATABASE root.sg1"); @@ -50,15 +60,6 @@ public void testTsFileDecompositionWithMods() throws Exception { TestUtils.executeNonQueryWithRetry( senderEnv, "CREATE ALIGNED TIMESERIES root.sg1.d3(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); - TestUtils.executeNonQueryWithRetry( - receiverEnv, "CREATE ALIGNED TIMESERIES root.sg1.d1(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); - TestUtils.executeNonQueryWithRetry( - receiverEnv, "CREATE TIMESERIES root.sg1.d2.s1 WITH DATATYPE=FLOAT"); - TestUtils.executeNonQueryWithRetry( - receiverEnv, "CREATE TIMESERIES root.sg1.d2.s2 WITH DATATYPE=FLOAT"); - TestUtils.executeNonQueryWithRetry( - receiverEnv, "CREATE ALIGNED TIMESERIES root.sg1.d3(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); - TestUtils.executeNonQueryWithRetry( senderEnv, "INSERT INTO root.sg1.d1(time, s1, s2, s3) ALIGNED VALUES (1, 1.0, 2.0, 3.0), (2, 1.1, 2.1, 3.1), (3, 1.2, 2.2, 3.2), (4, 1.3, 2.3, 3.3), (5, 1.4, 2.4, 3.4)"); @@ -76,6 +77,39 @@ public void testTsFileDecompositionWithMods() throws Exception { TestUtils.executeNonQueryWithRetry( senderEnv, "DELETE FROM root.sg1.d1.s1 WHERE time >= 2 AND time <= 4"); + TestUtils.executeNonQueryWithRetry( + senderEnv, + "INSERT INTO root.sg1.d3(time, s1, s2, s3) ALIGNED VALUES (1, 100.0, 200.0, 300.0), (2, 100.1, 200.1, 300.1), (3, 100.2, 200.2, 300.2)"); + + TestUtils.executeNonQueryWithRetry( + senderEnv, "CREATE ALIGNED TIMESERIES root.sg1.d4(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); + String s = "INSERT INTO root.sg1.d4(time, s1, s2, s3) ALIGNED VALUES "; + StringBuilder insertBuilder = new StringBuilder(s); + for (int i = 1; i <= 11000; i++) { + insertBuilder + .append("(") + .append(i) + .append(",") + .append(1.0f) + .append(",") + .append(2.0f) + .append(",") + .append(3.0f) + .append(")"); + if (i % 100 != 0) { + insertBuilder.append(","); + } else { + TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder.toString()); + insertBuilder = new StringBuilder(s); + } + } + + TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + + TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d4.s1 WHERE time <= 10000"); + TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d4.s2 WHERE time > 1000"); + TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d4.s3 WHERE time <= 8000"); + TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d2.*"); TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d3.*"); @@ -83,11 +117,11 @@ public void testTsFileDecompositionWithMods() throws Exception { TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); executeNonQueryWithRetry( - senderEnv, - String.format( - "CREATE PIPE test_pipe WITH SOURCE ('mods.enable'='true') WITH CONNECTOR('ip'='%s', 'port'='%s', 'username'='root', 'format'='tablet')", - receiverEnv.getDataNodeWrapperList().get(0).getIp(), - receiverEnv.getDataNodeWrapperList().get(0).getPort())); + senderEnv, + String.format( + "CREATE PIPE test_pipe WITH SOURCE ('mods.enable'='true') WITH CONNECTOR('ip'='%s', 'port'='%s', 'format'='tablet')", + receiverEnv.getDataNodeWrapperList().get(0).getIp(), + receiverEnv.getDataNodeWrapperList().get(0).getPort())); HashSet results = new HashSet<>(); results.add("1,3.0,1.0,2.0,"); @@ -103,12 +137,27 @@ public void testTsFileDecompositionWithMods() throws Exception { results); TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "SELECT * FROM root.sg1.d2 ORDER BY time", "Time,root.sg1.d2.s1,root.sg1.d2.s2,", Collections.emptySet()); + receiverEnv, + "SELECT * FROM root.sg1.d2 ORDER BY time", + "Time,", + Collections.emptySet()); TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "SELECT * FROM root.sg1.d3 ORDER BY time", "Time,root.sg1.d3.s3,root.sg1.d3.s1,root.sg1.d3.s2,", Collections.emptySet()); + receiverEnv, + "SELECT * FROM root.sg1.d3 ORDER BY time", + "Time,", + Collections.emptySet()); TestUtils.assertDataEventuallyOnEnv( - receiverEnv, "SELECT s1 FROM root.sg1.d1 WHERE time >= 2 AND time <= 4", "Time,root.sg1.d1.s1,", Collections.emptySet()); + receiverEnv, + "SELECT s1 FROM root.sg1.d1 WHERE time >= 2 AND time <= 4", + "Time,root.sg1.d1.s1,", + Collections.emptySet()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(**) FROM root.sg1.d4", + "COUNT(root.sg1.d4.s3),COUNT(root.sg1.d4.s1),COUNT(root.sg1.d4.s2),", + Collections.singleton("3000,1000,1000,")); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java index df0548f2fa79..794fce449d8b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java @@ -38,6 +38,8 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.TsFileSequenceReader; @@ -90,6 +92,7 @@ public class TsFileInsertionEventScanParser extends TsFileInsertionEventParser { private final Map measurementIndexMap = new HashMap<>(); private int lastIndex = -1; private ChunkHeader firstChunkHeader4NextSequentialValueChunks; + private long firstChunkHeaderOffset4NextSequentialValueChunks = -1; private byte lastMarker = Byte.MIN_VALUE; @@ -128,7 +131,7 @@ public TsFileInsertionEventScanParser( PipeDataNodeResourceManager.memory() .forceAllocateForTabletWithRetry(currentModifications.ramBytesUsed()); - tsFileSequenceReader = new TsFileSequenceReader(tsFile.getAbsolutePath(), false, false); + tsFileSequenceReader = new TsFileSequenceReader(tsFile.getAbsolutePath(), true, true); tsFileSequenceReader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1); prepareData(); @@ -335,7 +338,8 @@ private void putValueToColumns(final BatchData data, final Tablet tablet, final if (data.getDataType() == TSDataType.VECTOR) { for (int i = 0; i < tablet.getSchemas().size(); ++i) { final TsPrimitiveType primitiveType = data.getVector()[i]; - if (Objects.isNull(primitiveType) || ModsOperationUtil.isDelete(i, modsInfos.get(i))) { + if (Objects.isNull(primitiveType) + || ModsOperationUtil.isDelete(data.currentTime(), modsInfos.get(i))) { switch (tablet.getSchemas().get(i).getType()) { case TEXT: case BLOB: @@ -431,7 +435,7 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { // Notice that the data in one chunk group is either aligned or non-aligned // There is no need to consider non-aligned chunks when there are value chunks currentIsMultiPage = marker == MetaMarker.CHUNK_HEADER; - + long currentChunkHeaderOffset = tsFileSequenceReader.position() - 1; chunkHeader = tsFileSequenceReader.readChunkHeader(marker); final long nextMarkerOffset = @@ -465,7 +469,11 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { new Chunk( chunkHeader, tsFileSequenceReader.readChunk(-1, chunkHeader.getDataSize())); // Skip the chunk if it is fully deleted by mods - final Statistics statistics = chunk.getChunkStatistic(); + final Statistics statistics = + findNonAlignedChunkStatistics( + tsFileSequenceReader.getIChunkMetadataList( + currentDevice, chunkHeader.getMeasurementID()), + currentChunkHeaderOffset); if (statistics != null && ModsOperationUtil.isAllDeletedByMods( currentDevice, @@ -493,7 +501,9 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { case MetaMarker.VALUE_CHUNK_HEADER: case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER: { + long currentChunkHeaderOffset = -1; if (Objects.isNull(firstChunkHeader4NextSequentialValueChunks)) { + currentChunkHeaderOffset = tsFileSequenceReader.position() - 1; chunkHeader = tsFileSequenceReader.readChunkHeader(marker); final long nextMarkerOffset = @@ -542,18 +552,25 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { lastIndex = valueIndex; if (needReturn) { firstChunkHeader4NextSequentialValueChunks = chunkHeader; + firstChunkHeaderOffset4NextSequentialValueChunks = currentChunkHeaderOffset; return; } } else { chunkHeader = firstChunkHeader4NextSequentialValueChunks; + currentChunkHeaderOffset = firstChunkHeaderOffset4NextSequentialValueChunks; firstChunkHeader4NextSequentialValueChunks = null; + firstChunkHeaderOffset4NextSequentialValueChunks = -1; } Chunk chunk = new Chunk( chunkHeader, tsFileSequenceReader.readChunk(-1, chunkHeader.getDataSize())); // Skip the chunk if it is fully deleted by mods - final Statistics statistics = chunk.getChunkStatistic(); + final Statistics statistics = + findAlignedChunkStatistics( + tsFileSequenceReader.getIChunkMetadataList( + currentDevice, chunkHeader.getMeasurementID()), + currentChunkHeaderOffset); if (statistics != null && ModsOperationUtil.isAllDeletedByMods( currentDevice, @@ -635,4 +652,32 @@ public void close() { allocatedMemoryBlockForChunk.close(); } } + + private Statistics findAlignedChunkStatistics( + List metadataList, long currentChunkHeaderOffset) { + for (IChunkMetadata metadata : metadataList) { + if (!(metadata instanceof AlignedChunkMetadata)) { + continue; + } + List list = ((AlignedChunkMetadata) metadata).getValueChunkMetadataList(); + for (IChunkMetadata m : list) { + if (m.getOffsetOfChunkHeader() == currentChunkHeaderOffset) { + return m.getStatistics(); + } + } + break; + } + return null; + } + + private Statistics findNonAlignedChunkStatistics( + List metadataList, long currentChunkHeaderOffset) { + for (IChunkMetadata metadata : metadataList) { + if (metadata.getOffsetOfChunkHeader() == currentChunkHeaderOffset) { + // found the corresponding chunk metadata + return metadata.getStatistics(); + } + } + return null; + } } From b65bef8f0d7a3345781fb0a928edf2cfe0cc7a54 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 14 Oct 2025 10:51:19 +0800 Subject: [PATCH 11/31] update IT --- .../IoTDBPipeTsFileDecompositionWithModsIT.java | 2 +- .../IoTDBPipeTsFileDecompositionWithModsIT.java | 16 +++------------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index 061197b25709..570007a9b80c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -90,4 +90,4 @@ public void testTsFileDecompositionWithMods() { Collections.emptySet(), "sg1"); } -} \ No newline at end of file +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index e998c63a9d69..079e263d6090 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -19,7 +19,6 @@ package org.apache.iotdb.pipe.it.dual.treemodel.manual; -import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; @@ -40,10 +39,7 @@ public class IoTDBPipeTsFileDecompositionWithModsIT extends AbstractPipeDualTree @Override protected void setupConfig() { super.setupConfig(); - receiverEnv - .getConfig() - .getCommonConfig() - .setAutoCreateSchemaEnabled(true); + receiverEnv.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true); } @Test @@ -137,16 +133,10 @@ public void testTsFileDecompositionWithMods() throws Exception { results); TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "SELECT * FROM root.sg1.d2 ORDER BY time", - "Time,", - Collections.emptySet()); + receiverEnv, "SELECT * FROM root.sg1.d2 ORDER BY time", "Time,", Collections.emptySet()); TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "SELECT * FROM root.sg1.d3 ORDER BY time", - "Time,", - Collections.emptySet()); + receiverEnv, "SELECT * FROM root.sg1.d3 ORDER BY time", "Time,", Collections.emptySet()); TestUtils.assertDataEventuallyOnEnv( receiverEnv, From 1113fda3b2460feceea1265d8fd0fa5582646ea1 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 14 Oct 2025 10:55:57 +0800 Subject: [PATCH 12/31] update TsFileInsertionEventScanParser --- .../scan/TsFileInsertionEventScanParser.java | 57 +++++++++---------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java index 794fce449d8b..a92e99fadf16 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java @@ -92,7 +92,6 @@ public class TsFileInsertionEventScanParser extends TsFileInsertionEventParser { private final Map measurementIndexMap = new HashMap<>(); private int lastIndex = -1; private ChunkHeader firstChunkHeader4NextSequentialValueChunks; - private long firstChunkHeaderOffset4NextSequentialValueChunks = -1; private byte lastMarker = Byte.MIN_VALUE; @@ -460,14 +459,6 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { break; } - if (chunkHeader.getDataSize() > allocatedMemoryBlockForChunk.getMemoryUsageInBytes()) { - PipeDataNodeResourceManager.memory() - .forceResize(allocatedMemoryBlockForChunk, chunkHeader.getDataSize()); - } - - Chunk chunk = - new Chunk( - chunkHeader, tsFileSequenceReader.readChunk(-1, chunkHeader.getDataSize())); // Skip the chunk if it is fully deleted by mods final Statistics statistics = findNonAlignedChunkStatistics( @@ -481,9 +472,19 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { statistics.getStartTime(), statistics.getEndTime(), currentModifications)) { + tsFileSequenceReader.position(nextMarkerOffset); break; } + if (chunkHeader.getDataSize() > allocatedMemoryBlockForChunk.getMemoryUsageInBytes()) { + PipeDataNodeResourceManager.memory() + .forceResize(allocatedMemoryBlockForChunk, chunkHeader.getDataSize()); + } + + Chunk chunk = + new Chunk( + chunkHeader, tsFileSequenceReader.readChunk(-1, chunkHeader.getDataSize())); + chunkReader = currentIsMultiPage ? new ChunkReader(chunk, filter) @@ -501,9 +502,8 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { case MetaMarker.VALUE_CHUNK_HEADER: case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER: { - long currentChunkHeaderOffset = -1; if (Objects.isNull(firstChunkHeader4NextSequentialValueChunks)) { - currentChunkHeaderOffset = tsFileSequenceReader.position() - 1; + long currentChunkHeaderOffset = tsFileSequenceReader.position() - 1; chunkHeader = tsFileSequenceReader.readChunkHeader(marker); final long nextMarkerOffset = @@ -515,6 +515,23 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { break; } + // Skip the chunk if it is fully deleted by mods + final Statistics statistics = + findAlignedChunkStatistics( + tsFileSequenceReader.getIChunkMetadataList( + currentDevice, chunkHeader.getMeasurementID()), + currentChunkHeaderOffset); + if (statistics != null + && ModsOperationUtil.isAllDeletedByMods( + currentDevice, + chunkHeader.getMeasurementID(), + statistics.getStartTime(), + statistics.getEndTime(), + currentModifications)) { + tsFileSequenceReader.position(nextMarkerOffset); + break; + } + // Increase value index final int valueIndex = measurementIndexMap.compute( @@ -552,34 +569,16 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { lastIndex = valueIndex; if (needReturn) { firstChunkHeader4NextSequentialValueChunks = chunkHeader; - firstChunkHeaderOffset4NextSequentialValueChunks = currentChunkHeaderOffset; return; } } else { chunkHeader = firstChunkHeader4NextSequentialValueChunks; - currentChunkHeaderOffset = firstChunkHeaderOffset4NextSequentialValueChunks; firstChunkHeader4NextSequentialValueChunks = null; - firstChunkHeaderOffset4NextSequentialValueChunks = -1; } Chunk chunk = new Chunk( chunkHeader, tsFileSequenceReader.readChunk(-1, chunkHeader.getDataSize())); - // Skip the chunk if it is fully deleted by mods - final Statistics statistics = - findAlignedChunkStatistics( - tsFileSequenceReader.getIChunkMetadataList( - currentDevice, chunkHeader.getMeasurementID()), - currentChunkHeaderOffset); - if (statistics != null - && ModsOperationUtil.isAllDeletedByMods( - currentDevice, - chunkHeader.getMeasurementID(), - statistics.getStartTime(), - statistics.getEndTime(), - currentModifications)) { - break; - } valueChunkSize += chunkHeader.getDataSize(); valueChunkList.add(chunk); From fdf567a833baa6c0097f7ea9307a5bb2892c04cb Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 14 Oct 2025 11:07:07 +0800 Subject: [PATCH 13/31] update TsFileInsertionEventScanParser --- ...oTDBPipeTsFileDecompositionWithModsIT.java | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index 079e263d6090..3f5ab11f7f8c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -150,4 +150,134 @@ public void testTsFileDecompositionWithMods() throws Exception { "COUNT(root.sg1.d4.s3),COUNT(root.sg1.d4.s1),COUNT(root.sg1.d4.s2),", Collections.singleton("3000,1000,1000,")); } + + @Test + public void testTsFileDecompositionWithMods2() throws Exception { + TestUtils.executeNonQueryWithRetry(senderEnv, "CREATE DATABASE root.sg1"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "CREATE DATABASE root.sg1"); + + TestUtils.executeNonQueryWithRetry( + senderEnv, "CREATE ALIGNED TIMESERIES root.sg1.d1(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); + TestUtils.executeNonQueryWithRetry( + senderEnv, "CREATE TIMESERIES root.sg1.d2.s1 WITH DATATYPE=FLOAT"); + TestUtils.executeNonQueryWithRetry( + senderEnv, "CREATE TIMESERIES root.sg1.d2.s2 WITH DATATYPE=FLOAT"); + TestUtils.executeNonQueryWithRetry( + senderEnv, "CREATE ALIGNED TIMESERIES root.sg1.d3(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); + + TestUtils.executeNonQueryWithRetry( + senderEnv, + "INSERT INTO root.sg1.d1(time, s1, s2, s3) ALIGNED VALUES (1, 1.0, 2.0, 3.0), (2, 1.1, 2.1, 3.1), (3, 1.2, 2.2, 3.2), (4, 1.3, 2.3, 3.3), (5, 1.4, 2.4, 3.4)"); + + TestUtils.executeNonQueryWithRetry( + senderEnv, + "INSERT INTO root.sg1.d2(time, s1, s2) VALUES (1, 10.0, 20.0), (2, 10.1, 20.1), (3, 10.2, 20.2), (4, 10.3, 20.3), (5, 10.4, 20.4)"); + + TestUtils.executeNonQueryWithRetry( + senderEnv, + "INSERT INTO root.sg1.d3(time, s1, s2, s3) ALIGNED VALUES (1, 100.0, 200.0, 300.0), (2, 100.1, 200.1, 300.1), (3, 100.2, 200.2, 300.2)"); + + TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + + TestUtils.executeNonQueryWithRetry( + senderEnv, "DELETE FROM root.sg1.d1.s1 WHERE time >= 2 AND time <= 4"); + + TestUtils.executeNonQueryWithRetry( + senderEnv, + "INSERT INTO root.sg1.d3(time, s1, s2, s3) ALIGNED VALUES (1, 100.0, 200.0, 300.0), (2, 100.1, 200.1, 300.1), (3, 100.2, 200.2, 300.2)"); + + TestUtils.executeNonQueryWithRetry( + senderEnv, "CREATE ALIGNED TIMESERIES root.sg1.d4(s1 FLOAT, s2 FLOAT, s3 FLOAT)"); + String s = "INSERT INTO root.sg1.d4(time, s1, s2, s3) ALIGNED VALUES "; + StringBuilder insertBuilder = new StringBuilder(s); + for (int i = 1; i <= 11000; i++) { + insertBuilder + .append("(") + .append(i) + .append(",") + .append(1.0f) + .append(",") + .append(2.0f) + .append(",") + .append(3.0f) + .append(")"); + if (i % 100 != 0) { + insertBuilder.append(","); + } else { + TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder.toString()); + insertBuilder = new StringBuilder(s); + } + } + + TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + + TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d4.s1 WHERE time <= 10000"); + TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d4.s2 WHERE time > 1000"); + TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d4.s3 WHERE time <= 8000"); + + TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d2.*"); + + TestUtils.executeNonQueryWithRetry(senderEnv, "DELETE FROM root.sg1.d3.*"); + + TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + + executeNonQueryWithRetry( + senderEnv, + String.format( + "CREATE PIPE test_pipe1 WITH SOURCE ('mods.enable'='true','path'='root.sg1.d1.**') WITH CONNECTOR('ip'='%s', 'port'='%s', 'format'='tablet')", + receiverEnv.getDataNodeWrapperList().get(0).getIp(), + receiverEnv.getDataNodeWrapperList().get(0).getPort())); + + executeNonQueryWithRetry( + senderEnv, + String.format( + "CREATE PIPE test_pipe2 WITH SOURCE ('mods.enable'='true','path'='root.sg1.d2.**') WITH CONNECTOR('ip'='%s', 'port'='%s', 'format'='tablet')", + receiverEnv.getDataNodeWrapperList().get(0).getIp(), + receiverEnv.getDataNodeWrapperList().get(0).getPort())); + + executeNonQueryWithRetry( + senderEnv, + String.format( + "CREATE PIPE test_pipe3 WITH SOURCE ('mods.enable'='true','path'='root.sg1.d3.**') WITH CONNECTOR('ip'='%s', 'port'='%s', 'format'='tablet')", + receiverEnv.getDataNodeWrapperList().get(0).getIp(), + receiverEnv.getDataNodeWrapperList().get(0).getPort())); + + executeNonQueryWithRetry( + senderEnv, + String.format( + "CREATE PIPE test_pipe4 WITH SOURCE ('mods.enable'='true','path'='root.sg1.d4.**') WITH CONNECTOR('ip'='%s', 'port'='%s', 'format'='tablet')", + receiverEnv.getDataNodeWrapperList().get(0).getIp(), + receiverEnv.getDataNodeWrapperList().get(0).getPort())); + + HashSet results = new HashSet<>(); + results.add("1,3.0,1.0,2.0,"); + results.add("2,3.1,null,2.1,"); + results.add("3,3.2,null,2.2,"); + results.add("4,3.3,null,2.3,"); + results.add("5,3.4,1.4,2.4,"); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT * FROM root.sg1.d1 ORDER BY time", + "Time,root.sg1.d1.s3,root.sg1.d1.s1,root.sg1.d1.s2,", + results); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "SELECT * FROM root.sg1.d2 ORDER BY time", "Time,", Collections.emptySet()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "SELECT * FROM root.sg1.d3 ORDER BY time", "Time,", Collections.emptySet()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT s1 FROM root.sg1.d1 WHERE time >= 2 AND time <= 4", + "Time,root.sg1.d1.s1,", + Collections.emptySet()); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(**) FROM root.sg1.d4", + "COUNT(root.sg1.d4.s3),COUNT(root.sg1.d4.s1),COUNT(root.sg1.d4.s2),", + Collections.singleton("3000,1000,1000,")); + } } From 7e222df4fc3c9d5bb34709f35ea9574a077dc97a Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 14 Oct 2025 14:40:40 +0800 Subject: [PATCH 14/31] update TsFileInsertionEventTableParserTabletIterator --- .../it/dual/tablemodel/TableModelUtils.java | 33 +++++++ ...oTDBPipeTsFileDecompositionWithModsIT.java | 87 +++++++++++++++++++ ...sertionEventTableParserTabletIterator.java | 46 +++++++--- 3 files changed, 155 insertions(+), 11 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java index 265548d80ccd..e03188d73762 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java @@ -115,6 +115,39 @@ public static void insertData( TestUtils.executeNonQueries(dataBaseName, BaseEnv.TABLE_SQL_DIALECT, baseEnv, list, null); } + public static void insertData( + final String dataBaseName, + final String tableName, + final int deviceStartIndex, + final int deviceEndIndex, + final int startInclusive, + final int endExclusive, + final BaseEnv baseEnv) { + List list = new ArrayList<>(endExclusive - startInclusive + 1); + for (int deviceIndex = deviceStartIndex; deviceIndex < deviceEndIndex; ++deviceIndex) { + for (int i = startInclusive; i < endExclusive; ++i) { + list.add( + String.format( + "insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)", + tableName, + deviceIndex, + deviceIndex, + deviceIndex, + deviceIndex, + i, + i, + i, + i, + i, + i, + getDateStr(i), + i, + i)); + } + } + TestUtils.executeNonQueries(dataBaseName, BaseEnv.TABLE_SQL_DIALECT, baseEnv, list, null); + } + public static void insertData( final String dataBaseName, final String tableName, diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index 570007a9b80c..46b02dfdccf3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -45,6 +45,11 @@ public void testTsFileDecompositionWithMods() { TableModelUtils.insertData("sg1", "table1", 1, 6, senderEnv); + TableModelUtils.createDataBaseAndTable(senderEnv, "table1", "sg2"); + for (int i = 1; i <= 110; i++) { + TableModelUtils.insertData("sg2", "table1", 10, 15, (i - 1) * 100, i * 100, senderEnv); + } + executeNonQueryWithRetry(senderEnv, "FLUSH"); executeNonQueryWithRetry( @@ -63,6 +68,46 @@ public void testTsFileDecompositionWithMods() { "sg1", "table"); + executeNonQueryWithRetry( + senderEnv, + "DELETE FROM table1 WHERE time >= 0 AND time < 10000 AND s0 ='t10' AND s1='t10' AND s2='t10' AND s3='t10'", + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + "sg2", + "table"); + + executeNonQueryWithRetry( + senderEnv, + "DELETE FROM table1 WHERE time >= 0 AND time <= 11000 AND s0 ='t11' AND s1='t11' AND s2='t11' AND s3='t11'", + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + "sg2", + "table"); + + executeNonQueryWithRetry( + senderEnv, + "DELETE FROM table1 WHERE time >= 5000 AND time < 10100 AND s0 ='t12' AND s1='t12' AND s2='t12' AND s3='t12'", + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + "sg2", + "table"); + + executeNonQueryWithRetry( + senderEnv, + "DELETE FROM table1 WHERE time >= 0 AND time < 10000 AND s0 ='t13' AND s1='t13' AND s2='t13' AND s3='t13'", + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + "sg2", + "table"); + + executeNonQueryWithRetry( + senderEnv, + "DELETE FROM table1 WHERE time >= 10000 AND time <= 11000 AND s0 ='t14' AND s1='t14' AND s2='t14' AND s3='t14'", + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + "sg2", + "table"); + executeNonQueryWithRetry(senderEnv, "FLUSH"); executeNonQueryWithRetry( @@ -89,5 +134,47 @@ public void testTsFileDecompositionWithMods() { "s4,", Collections.emptySet(), "sg1"); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t10' AND s1='t10' AND s2='t10' AND s3='t10'", + "count,", + Collections.singleton("1000,"), + "sg2"); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t11' AND s1='t11' AND s2='t11' AND s3='t11'", + "count,", + Collections.singleton("0,"), + "sg2"); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t11' AND s1='t11' AND s2='t11' AND s3='t11'", + "count,", + Collections.singleton("0,"), + "sg2"); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t12' AND s1='t12' AND s2='t12' AND s3='t12'", + "count,", + Collections.singleton("5900,"), + "sg2"); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t13' AND s1='t13' AND s2='t13' AND s3='t13'", + "count,", + Collections.singleton("1000,"), + "sg2"); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t14' AND s1='t14' AND s2='t14' AND s3='t14'", + "count,", + Collections.singleton("10000,"), + "sg2"); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java index 1dae9aacd6e4..87e818b07ad3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java @@ -222,14 +222,32 @@ public boolean hasNext() { "Table model tsfile parsing does not support this type of ChunkMeta"); } - if (ModsOperationUtil.isAllDeletedByMods( - pair.getLeft(), - iChunkMetadata.getMeasurementUid(), - alignedChunkMetadata.getStartTime(), - alignedChunkMetadata.getEndTime(), - modifications)) { + boolean isDelete = false; + if (isDelete = + ModsOperationUtil.isAllDeletedByMods( + pair.getLeft(), + iChunkMetadata.getMeasurementUid(), + alignedChunkMetadata.getStartTime(), + alignedChunkMetadata.getEndTime(), + modifications)) { iChunkMetadataIterator.remove(); } + System.out.println( + "deviceID: " + + pair.getLeft() + + ", measurement: " + + iChunkMetadata.getMeasurementUid() + + ", startTime: " + + alignedChunkMetadata.getStartTime() + + ", endTime: " + + alignedChunkMetadata.getEndTime() + + ", isDelete: " + + isDelete); + } + + if (alignedChunkMetadata.getValueChunkMetadataList().isEmpty()) { + chunkMetadataIterator.remove(); + continue; } size += @@ -337,9 +355,10 @@ private Tablet buildNextTablet() { break; } - tablet.addTimestamp(rowIndex, batchData.currentTime()); - fillMeasurementValueColumns(batchData, tablet, rowIndex); - fillDeviceIdColumns(deviceID, tablet, rowIndex); + if (fillMeasurementValueColumns(batchData, tablet, rowIndex)) { + tablet.addTimestamp(rowIndex, batchData.currentTime()); + fillDeviceIdColumns(deviceID, tablet, rowIndex); + } } if (batchData != null) { @@ -420,14 +439,16 @@ private void initChunkReader(final AbstractAlignedChunkMetadata alignedChunkMeta ModsOperationUtil.initializeMeasurementMods(deviceID, measurementList, modifications); } - private void fillMeasurementValueColumns( + private boolean fillMeasurementValueColumns( final BatchData data, final Tablet tablet, final int rowIndex) { final TsPrimitiveType[] primitiveTypes = data.getVector(); + boolean needFillTime = false; for (int i = deviceIdSize, size = dataTypeList.size(); i < size; i++) { final TsPrimitiveType primitiveType = primitiveTypes[i - deviceIdSize]; + boolean isDelete = false; if (primitiveType == null - || ModsOperationUtil.isDelete(data.currentTime(), modsInfoList.get(i))) { + || (isDelete = ModsOperationUtil.isDelete(data.currentTime(), modsInfoList.get(i)))) { switch (dataTypeList.get(i)) { case TEXT: case BLOB: @@ -435,8 +456,10 @@ private void fillMeasurementValueColumns( tablet.addValue(rowIndex, i, Binary.EMPTY_VALUE.getValues()); } tablet.getBitMaps()[i].mark(rowIndex); + needFillTime = needFillTime || !isDelete; continue; } + needFillTime = true; switch (dataTypeList.get(i)) { case BOOLEAN: @@ -471,6 +494,7 @@ private void fillMeasurementValueColumns( throw new UnSupportedDataTypeException("UnSupported" + primitiveType.getDataType()); } } + return needFillTime; } private void fillDeviceIdColumns( From 051b57615065f9d9e5f6c2c3f0c71058cc8644f3 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 14 Oct 2025 14:58:03 +0800 Subject: [PATCH 15/31] fix --- ...eInsertionEventQueryParserTabletIterator.java | 12 +++++++++--- .../scan/TsFileInsertionEventScanParser.java | 16 ++++++++++++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java index 222fc33d1153..ceb63ac133bb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java @@ -175,22 +175,28 @@ private Tablet buildNextTablet() throws IOException { final int rowIndex = tablet.getRowSize(); - tablet.addTimestamp(rowIndex, rowRecord.getTimestamp()); + boolean isNeedFillTime = false; final List fields = rowRecord.getFields(); final int fieldSize = fields.size(); for (int i = 0; i < fieldSize; i++) { final Field field = fields.get(i); final String measurement = measurements.get(i); - + boolean isDeletedByMods = false; // Check if this value is deleted by mods if (field == null - || ModsOperationUtil.isDelete(rowRecord.getTimestamp(), measurementModsList.get(i))) { + || (isDeletedByMods = + ModsOperationUtil.isDelete(rowRecord.getTimestamp(), measurementModsList.get(i)))) { tablet.getBitMaps()[i].mark(rowIndex); + isNeedFillTime = isNeedFillTime || !isDeletedByMods; } else { tablet.addValue(measurement, rowIndex, field.getObjectValue(schemas.get(i).getType())); + isNeedFillTime = true; } } + if (isNeedFillTime) { + tablet.addTimestamp(rowIndex, rowRecord.getTimestamp()); + } if (tablet.getRowSize() == tablet.getMaxRowNumber()) { break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java index a92e99fadf16..b78975f3ee06 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java @@ -282,8 +282,9 @@ private Tablet getNextTablet() { final int rowIndex = tablet.getRowSize(); - tablet.addTimestamp(rowIndex, data.currentTime()); - putValueToColumns(data, tablet, rowIndex); + if (putValueToColumns(data, tablet, rowIndex)) { + tablet.addTimestamp(rowIndex, data.currentTime()); + } } data.next(); @@ -333,12 +334,14 @@ private void prepareData() throws IOException { } while (!data.hasCurrent()); } - private void putValueToColumns(final BatchData data, final Tablet tablet, final int rowIndex) { + private boolean putValueToColumns(final BatchData data, final Tablet tablet, final int rowIndex) { + boolean isNeedFillTime = false; if (data.getDataType() == TSDataType.VECTOR) { for (int i = 0; i < tablet.getSchemas().size(); ++i) { final TsPrimitiveType primitiveType = data.getVector()[i]; + boolean isDeleted = false; if (Objects.isNull(primitiveType) - || ModsOperationUtil.isDelete(data.currentTime(), modsInfos.get(i))) { + || (isDeleted = ModsOperationUtil.isDelete(data.currentTime(), modsInfos.get(i)))) { switch (tablet.getSchemas().get(i).getType()) { case TEXT: case BLOB: @@ -346,8 +349,11 @@ private void putValueToColumns(final BatchData data, final Tablet tablet, final tablet.addValue(rowIndex, i, Binary.EMPTY_VALUE.getValues()); } tablet.getBitMaps()[i].mark(rowIndex); + isNeedFillTime = isNeedFillTime || !isDeleted; continue; } + + isNeedFillTime = true; switch (tablet.getSchemas().get(i).getType()) { case BOOLEAN: tablet.addValue(rowIndex, i, primitiveType.getBoolean()); @@ -378,6 +384,7 @@ private void putValueToColumns(final BatchData data, final Tablet tablet, final } } } else { + isNeedFillTime = true; switch (tablet.getSchemas().get(0).getType()) { case BOOLEAN: tablet.addValue(rowIndex, 0, data.getBoolean()); @@ -407,6 +414,7 @@ private void putValueToColumns(final BatchData data, final Tablet tablet, final throw new UnSupportedDataTypeException("UnSupported" + data.getDataType()); } } + return isNeedFillTime; } private void moveToNextChunkReader() throws IOException, IllegalStateException { From e74f6692196949a996eb26a34ce2c0b6dbecd584 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 14 Oct 2025 15:06:38 +0800 Subject: [PATCH 16/31] update --- ...oTDBPipeTsFileDecompositionWithModsIT.java | 18 ++++++++ ...oTDBPipeTsFileDecompositionWithModsIT.java | 42 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index 46b02dfdccf3..b23121990845 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -38,6 +38,24 @@ @Category({MultiClusterIT2DualTableManualBasic.class}) public class IoTDBPipeTsFileDecompositionWithModsIT extends AbstractPipeTableModelDualManualIT { + /** + * Test IoTDB pipe handling TsFile decomposition with Mods (modification operations) in table + * model + * + *

Test scenario: 1. Create two storage groups sg1 and sg2, each containing table1 2. Insert + * small amount of data in sg1 (1-6 rows), insert large amount of data in sg2 (110 batches, 100 + * rows per batch) 3. Execute FLUSH operation to persist data to TsFile 4. Execute multiple DELETE + * operations on sg1, deleting data in time ranges 2-4 and 3-5 5. Execute multiple DELETE + * operations on sg2, deleting data matching specific conditions (s0-s3 field values) 6. Execute + * FLUSH operation again 7. Create pipe with mods enabled, synchronize data to receiver 8. Verify + * correctness of receiver data: - sg1 only retains time=1 data, time=2-4 data is correctly + * deleted - sg2 DELETE operation results meet expectations (t10 retains 1000 rows, t11 all + * deleted, t12 retains 5900 rows, etc.) + * + *

Test purpose: Verify that IoTDB pipe can correctly handle Mods (modification operations) in + * TsFile, ensuring DELETE operations can be correctly synchronized to the receiver and data + * consistency is guaranteed. + */ @Test public void testTsFileDecompositionWithMods() { TableModelUtils.createDataBaseAndTable(senderEnv, "table1", "sg1"); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index 3f5ab11f7f8c..df1508d1a6eb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -42,6 +42,24 @@ protected void setupConfig() { receiverEnv.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true); } + /** + * Test IoTDB pipe handling TsFile decomposition with Mods (modification operations) in tree model + * + *

Test scenario: 1. Create database root.sg1 with 4 devices: d1 (aligned timeseries), d2 + * (non-aligned timeseries), d3 (aligned timeseries), d4 (aligned timeseries) 2. Insert initial + * data into d1, d2, d3 3. Execute FLUSH operation to persist data to TsFile 4. Execute DELETE + * operation on d1.s1, deleting data in time range 2-4 5. Insert large amount of data into d4 + * (11000 records, inserted in batches) 6. Execute multiple DELETE operations on d4: - Delete s1 + * field data where time<=10000 - Delete s2 field data where time>1000 - Delete s3 field data + * where time<=8000 7. Delete all data from d2 and d3 8. Execute FLUSH operation again 9. Create + * pipe with mods enabled, synchronize data to receiver 10. Verify correctness of receiver data: - + * d1 s1 field is null in time range 2-4, other data is normal - d2 and d3 data is completely + * deleted - d4 DELETE operation results meet expectations for each field + * + *

Test purpose: Verify that IoTDB pipe can correctly handle Mods (modification operations) in + * TsFile under tree model, ensuring various DELETE operations can be correctly synchronized to + * the receiver and data consistency is guaranteed. + */ @Test public void testTsFileDecompositionWithMods() throws Exception { TestUtils.executeNonQueryWithRetry(senderEnv, "CREATE DATABASE root.sg1"); @@ -151,6 +169,30 @@ public void testTsFileDecompositionWithMods() throws Exception { Collections.singleton("3000,1000,1000,")); } + /** + * Test IoTDB pipe handling TsFile decomposition with Mods (modification operations) in tree model + * - Multi-pipe scenario + * + *

Test scenario: 1. Create database root.sg1 with 4 devices: d1 (aligned timeseries), d2 + * (non-aligned timeseries), d3 (aligned timeseries), d4 (aligned timeseries) 2. Insert initial + * data into d1, d2, d3 3. Execute FLUSH operation to persist data to TsFile 4. Execute DELETE + * operation on d1.s1, deleting data in time range 2-4 5. Insert large amount of data into d4 + * (11000 records, inserted in batches) 6. Execute multiple DELETE operations on d4: - Delete s1 + * field data where time<=10000 - Delete s2 field data where time>1000 - Delete s3 field data + * where time<=8000 7. Delete all data from d2 and d3 8. Execute FLUSH operation again 9. Create 4 + * independent pipes, each targeting different device paths: - test_pipe1: handles data for + * root.sg1.d1.** path - test_pipe2: handles data for root.sg1.d2.** path - test_pipe3: handles + * data for root.sg1.d3.** path - test_pipe4: handles data for root.sg1.d4.** path 10. Verify + * correctness of receiver data: - d1 s1 field is null in time range 2-4, other data is normal - + * d2 and d3 data is completely deleted - d4 DELETE operation results meet expectations for each + * field + * + *

Test purpose: Verify that IoTDB pipe can correctly handle Mods (modification operations) in + * TsFile under tree model through multiple independent pipes, ensuring DELETE operations for + * different paths can be correctly synchronized to the receiver and data consistency is + * guaranteed. The main difference from the first test method is using multiple pipes to handle + * data for different devices separately. + */ @Test public void testTsFileDecompositionWithMods2() throws Exception { TestUtils.executeNonQueryWithRetry(senderEnv, "CREATE DATABASE root.sg1"); From 1eb6745844f69db3034fe34acb46a47108853ff8 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 14 Oct 2025 19:04:49 +0800 Subject: [PATCH 17/31] update --- ...sertionEventTableParserTabletIterator.java | 30 +++++-------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java index 87e818b07ad3..ce2f4fe353d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java @@ -218,31 +218,17 @@ public boolean hasNext() { while (iChunkMetadataIterator.hasNext()) { IChunkMetadata iChunkMetadata = iChunkMetadataIterator.next(); if (iChunkMetadata == null) { - throw new PipeException( - "Table model tsfile parsing does not support this type of ChunkMeta"); + continue; } - boolean isDelete = false; - if (isDelete = - ModsOperationUtil.isAllDeletedByMods( - pair.getLeft(), - iChunkMetadata.getMeasurementUid(), - alignedChunkMetadata.getStartTime(), - alignedChunkMetadata.getEndTime(), - modifications)) { + if (ModsOperationUtil.isAllDeletedByMods( + pair.getLeft(), + iChunkMetadata.getMeasurementUid(), + alignedChunkMetadata.getStartTime(), + alignedChunkMetadata.getEndTime(), + modifications)) { iChunkMetadataIterator.remove(); } - System.out.println( - "deviceID: " - + pair.getLeft() - + ", measurement: " - + iChunkMetadata.getMeasurementUid() - + ", startTime: " - + alignedChunkMetadata.getStartTime() - + ", endTime: " - + alignedChunkMetadata.getEndTime() - + ", isDelete: " - + isDelete); } if (alignedChunkMetadata.getValueChunkMetadataList().isEmpty()) { @@ -356,8 +342,8 @@ private Tablet buildNextTablet() { } if (fillMeasurementValueColumns(batchData, tablet, rowIndex)) { - tablet.addTimestamp(rowIndex, batchData.currentTime()); fillDeviceIdColumns(deviceID, tablet, rowIndex); + tablet.addTimestamp(rowIndex, batchData.currentTime()); } } From b7b817c6f96276901e8595dc83f5688043840102 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 15 Oct 2025 11:42:53 +0800 Subject: [PATCH 18/31] modify isDelete function --- ...oTDBPipeTsFileDecompositionWithModsIT.java | 8 - .../tsfile/parser/util/ModsOperationUtil.java | 102 ++++- .../parser/util/ModsOperationUtilTest.java | 416 ++++++++++++++++++ 3 files changed, 502 insertions(+), 24 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtilTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index b23121990845..5a3ae45a81f7 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -94,14 +94,6 @@ public void testTsFileDecompositionWithMods() { "sg2", "table"); - executeNonQueryWithRetry( - senderEnv, - "DELETE FROM table1 WHERE time >= 0 AND time <= 11000 AND s0 ='t11' AND s1='t11' AND s2='t11' AND s3='t11'", - SessionConfig.DEFAULT_USER, - SessionConfig.DEFAULT_PASSWORD, - "sg2", - "table"); - executeNonQueryWithRetry( senderEnv, "DELETE FROM table1 WHERE time >= 5000 AND time < 10100 AND s0 ='t12' AND s1='t12' AND s2='t12' AND s3='t12'", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java index 795577cdfe45..b0063fb0cc59 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java @@ -176,28 +176,98 @@ public static boolean isDelete(long time, ModsInfo modsInfo) { return false; } - // Search from current index - for (int i = currentIndex; i < mods.size(); i++) { - final ModEntry mod = mods.get(i); - final long modStartTime = mod.getTimeRange().getMin(); - final long modEndTime = mod.getTimeRange().getMax(); - - if (time < modStartTime) { - // Current time is before mod start time, update index and return false - modsInfo.setCurrentIndex(i); - return false; - } else if (time <= modEndTime) { - // Current time is within mod time range, update index and return true - modsInfo.setCurrentIndex(i); + // First, try to use the current index if it's valid + if (currentIndex < mods.size()) { + final ModEntry currentMod = mods.get(currentIndex); + final long currentModStartTime = currentMod.getTimeRange().getMin(); + final long currentModEndTime = currentMod.getTimeRange().getMax(); + + if (time < currentModStartTime) { + // Time is before current mod, need to search backwards + return searchAndCheckMod(mods, time, 0, modsInfo); + } else if (time <= currentModEndTime) { + // Time is within current mod range, return true return true; + } else { + // Time is after current mod, need to search forwards + return searchAndCheckMod(mods, time, currentIndex + 1, modsInfo); } - // If time > modEndTime, continue to next mod + } else { + // Current index is beyond array bounds, all mods have been processed + clearModsAndReset(modsInfo); + return false; + } + } + + /** + * Search for a mod using binary search and check if the time point is deleted + * + * @param mods sorted list of mods + * @param time time point to search for + * @param startIndex starting index for search + * @param modsInfo mods information to update + * @return true if data is deleted, false otherwise + */ + private static boolean searchAndCheckMod( + List mods, long time, int startIndex, ModsInfo modsInfo) { + int searchIndex = binarySearchMods(mods, time, startIndex); + if (searchIndex >= mods.size()) { + // All mods checked, clear mods list and reset index to 0 + clearModsAndReset(modsInfo); + return false; + } + + final ModEntry foundMod = mods.get(searchIndex); + final long foundModStartTime = foundMod.getTimeRange().getMin(); + final long foundModEndTime = foundMod.getTimeRange().getMax(); + + if (time < foundModStartTime) { + modsInfo.setCurrentIndex(searchIndex); + return false; + } else if (time <= foundModEndTime) { + modsInfo.setCurrentIndex(searchIndex); + return true; + } else { + modsInfo.setCurrentIndex(searchIndex + 1); + return false; } + } - // All mods checked, clear mods list and reset index to 0 + /** + * Clear mods list and reset index to 0 + * + * @param modsInfo mods information to update + */ + private static void clearModsAndReset(ModsInfo modsInfo) { modsInfo.setMods(Collections.emptyList()); modsInfo.setCurrentIndex(0); - return false; + } + + /** + * Binary search to find the first mod that might contain the given time point. Returns the index + * of the first mod where modStartTime <= time, or mods.size() if no such mod exists. + * + * @param mods sorted list of mods + * @param time time point to search for + * @param startIndex starting index for search (current index) + * @return index of the first potential mod, or mods.size() if none found + */ + private static int binarySearchMods(List mods, long time, int startIndex) { + int left = startIndex; + int right = mods.size(); + + while (left < right) { + int mid = left + (right - left) / 2; + final long max = mods.get(mid).getTimeRange().getMax(); + + if (max < time) { + left = mid + 1; + } else { + right = mid; + } + } + + return left; } /** Mods information wrapper class, containing mods list and current index */ diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtilTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtilTest.java new file mode 100644 index 000000000000..d0a4037fb90d --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtilTest.java @@ -0,0 +1,416 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.event.common.tsfile.parser.util; + +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; + +import org.apache.tsfile.read.common.TimeRange; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class ModsOperationUtilTest { + + @Test + public void testIsDeleteWithBinarySearch() { + // Create test mods with time ranges: [10, 20], [30, 40], [50, 60], [70, 80] + List mods = createTestMods(new long[][] {{10, 20}, {30, 40}, {50, 60}, {70, 80}}); + + ModsOperationUtil.ModsInfo modsInfo = new ModsOperationUtil.ModsInfo(mods, 0); + + // Test cases + // Time 5: before first mod, should return false + assertFalse(ModsOperationUtil.isDelete(5, modsInfo)); + assertEquals(0, modsInfo.getCurrentIndex()); + + // Time 15: within first mod [10, 20], should return true + assertTrue(ModsOperationUtil.isDelete(15, modsInfo)); + assertEquals(0, modsInfo.getCurrentIndex()); + + // Time 25: between first and second mod, should return false + assertFalse(ModsOperationUtil.isDelete(25, modsInfo)); + assertEquals(1, modsInfo.getCurrentIndex()); + + // Time 35: within second mod [30, 40], should return true + assertTrue(ModsOperationUtil.isDelete(35, modsInfo)); + assertEquals(1, modsInfo.getCurrentIndex()); + + // Time 45: between second and third mod, should return false + assertFalse(ModsOperationUtil.isDelete(45, modsInfo)); + assertEquals(2, modsInfo.getCurrentIndex()); + + // Time 55: within third mod [50, 60], should return true + assertTrue(ModsOperationUtil.isDelete(55, modsInfo)); + assertEquals(2, modsInfo.getCurrentIndex()); + + // Time 65: between third and fourth mod, should return false + assertFalse(ModsOperationUtil.isDelete(65, modsInfo)); + assertEquals(3, modsInfo.getCurrentIndex()); + + // Time 75: within fourth mod [70, 80], should return true + assertTrue(ModsOperationUtil.isDelete(75, modsInfo)); + assertEquals(3, modsInfo.getCurrentIndex()); + + // Time 85: after last mod, should return false and clear mods + assertFalse(ModsOperationUtil.isDelete(85, modsInfo)); + assertTrue(modsInfo.getMods().isEmpty()); + assertEquals(0, modsInfo.getCurrentIndex()); + } + + @Test + public void testIsDeleteWithEmptyMods() { + ModsOperationUtil.ModsInfo modsInfo = new ModsOperationUtil.ModsInfo(new ArrayList<>(), 0); + + // Should return false for any time when mods is empty + assertFalse(ModsOperationUtil.isDelete(100, modsInfo)); + } + + @Test + public void testIsDeleteWithNullModsInfo() { + // Should return false when modsInfo is null + assertFalse(ModsOperationUtil.isDelete(100, null)); + } + + @Test + public void testIsDeleteWithNegativeIndex() { + List mods = createTestMods(new long[][] {{10, 20}}); + ModsOperationUtil.ModsInfo modsInfo = new ModsOperationUtil.ModsInfo(mods, -1); + + // Should return false when currentIndex is negative + assertFalse(ModsOperationUtil.isDelete(15, modsInfo)); + } + + @Test + public void testIsDeleteWithSingleMod() { + List mods = createTestMods(new long[][] {{10, 20}}); + ModsOperationUtil.ModsInfo modsInfo = new ModsOperationUtil.ModsInfo(mods, 0); + + // Time before mod + assertFalse(ModsOperationUtil.isDelete(5, modsInfo)); + assertEquals(0, modsInfo.getCurrentIndex()); + + // Time within mod + assertTrue(ModsOperationUtil.isDelete(15, modsInfo)); + assertEquals(0, modsInfo.getCurrentIndex()); + + // Time after mod + assertFalse(ModsOperationUtil.isDelete(25, modsInfo)); + assertTrue(modsInfo.getMods().isEmpty()); + assertEquals(0, modsInfo.getCurrentIndex()); + } + + @Test + public void testIsDeleteWithOverlappingMods() { + // Create overlapping mods: [10, 30], [20, 40], [30, 50] + List mods = createTestMods(new long[][] {{10, 30}, {20, 40}, {30, 50}}); + + ModsOperationUtil.ModsInfo modsInfo = new ModsOperationUtil.ModsInfo(mods, 0); + + // Time 15: within first mod + assertTrue(ModsOperationUtil.isDelete(15, modsInfo)); + assertEquals(0, modsInfo.getCurrentIndex()); + + // Time 25: within both first and second mod, should find first one + assertTrue(ModsOperationUtil.isDelete(25, modsInfo)); + assertEquals(0, modsInfo.getCurrentIndex()); + + // Time 35: within second and third mod, should find second one + assertTrue(ModsOperationUtil.isDelete(35, modsInfo)); + assertEquals(1, modsInfo.getCurrentIndex()); + + // Time 45: within third mod + assertTrue(ModsOperationUtil.isDelete(45, modsInfo)); + assertEquals(2, modsInfo.getCurrentIndex()); + } + + @Test + public void testBinarySearchMods() { + // Create test mods with time ranges: [10, 20], [30, 40], [50, 60], [70, 80] + List mods = createTestMods(new long[][] {{10, 20}, {30, 40}, {50, 60}, {70, 80}}); + + // Test binary search from start index 0 + // Time 5: before all mods, should return 0 (first mod) + assertEquals(0, binarySearchMods(mods, 5, 0)); + + // Time 15: within first mod [10, 20], should return 0 + assertEquals(0, binarySearchMods(mods, 15, 0)); + + // Time 25: between first and second mod, should return 1 + assertEquals(1, binarySearchMods(mods, 25, 0)); + + // Time 35: within second mod [30, 40], should return 1 + assertEquals(1, binarySearchMods(mods, 35, 0)); + + // Time 45: between second and third mod, should return 2 + assertEquals(2, binarySearchMods(mods, 45, 0)); + + // Time 55: within third mod [50, 60], should return 2 + assertEquals(2, binarySearchMods(mods, 55, 0)); + + // Time 65: between third and fourth mod, should return 3 + assertEquals(3, binarySearchMods(mods, 65, 0)); + + // Time 75: within fourth mod [70, 80], should return 3 + assertEquals(3, binarySearchMods(mods, 75, 0)); + + // Time 85: after all mods, should return 4 (mods.size()) + assertEquals(4, binarySearchMods(mods, 85, 0)); + } + + @Test + public void testBinarySearchModsWithStartIndex() { + // Create test mods with time ranges: [10, 20], [30, 40], [50, 60], [70, 80] + List mods = createTestMods(new long[][] {{10, 20}, {30, 40}, {50, 60}, {70, 80}}); + + // Test binary search starting from index 2 + // Time 15: before start index, should return 2 (start index) + assertEquals(2, binarySearchMods(mods, 15, 2)); + + // Time 35: before start index, should return 2 (start index) + assertEquals(2, binarySearchMods(mods, 35, 2)); + + // Time 55: within mod at index 2, should return 2 + assertEquals(2, binarySearchMods(mods, 55, 2)); + + // Time 65: between mods at index 2 and 3, should return 3 + assertEquals(3, binarySearchMods(mods, 65, 2)); + + // Time 75: within mod at index 3, should return 3 + assertEquals(3, binarySearchMods(mods, 75, 2)); + + // Time 85: after all mods, should return 4 (mods.size()) + assertEquals(4, binarySearchMods(mods, 85, 2)); + } + + @Test + public void testBinarySearchModsWithOverlappingRanges() { + // Create overlapping mods: [10, 30], [20, 40], [30, 50] + List mods = createTestMods(new long[][] {{10, 30}, {20, 40}, {30, 50}}); + + // Time 15: within first mod, should return 0 + assertEquals(0, binarySearchMods(mods, 15, 0)); + + // Time 25: within first and second mod, should return 0 (first match) + assertEquals(0, binarySearchMods(mods, 25, 0)); + + // Time 35: within second and third mod, should return 1 (first match from start) + assertEquals(1, binarySearchMods(mods, 35, 0)); + + // Time 45: within third mod, should return 2 + assertEquals(2, binarySearchMods(mods, 45, 0)); + } + + @Test + public void testBinarySearchModsWithEmptyList() { + List mods = new ArrayList<>(); + + // Should return 0 for any time when mods is empty + assertEquals(0, binarySearchMods(mods, 100, 0)); + } + + @Test + public void testBinarySearchModsWithSingleMod() { + List mods = createTestMods(new long[][] {{10, 20}}); + + // Time 5: before mod, should return 0 + assertEquals(0, binarySearchMods(mods, 5, 0)); + + // Time 15: within mod, should return 0 + assertEquals(0, binarySearchMods(mods, 15, 0)); + + // Time 25: after mod, should return 1 + assertEquals(1, binarySearchMods(mods, 25, 0)); + } + + @Test + public void testBinarySearchModsWithExactBoundaries() { + // Create mods with exact boundaries: [10, 20], [20, 30], [30, 40] + List mods = createTestMods(new long[][] {{10, 20}, {20, 30}, {30, 40}}); + + // Time 20: at boundary, first mod [10, 20] has endTime=20 >= 20, should return 0 + assertEquals(0, binarySearchMods(mods, 20, 0)); + + // Time 30: at boundary, second mod [20, 30] has endTime=30 >= 30, should return 1 + assertEquals(1, binarySearchMods(mods, 30, 0)); + } + + @Test + public void testBinarySearchModsWithMinBoundaries() { + // Create mods: [10, 20], [30, 40], [50, 60] + List mods = createTestMods(new long[][] {{10, 20}, {30, 40}, {50, 60}}); + + // Time 10: exactly at first mod's min, should return 0 + assertEquals(0, binarySearchMods(mods, 10, 0)); + + // Time 30: exactly at second mod's min, should return 1 + assertEquals(1, binarySearchMods(mods, 30, 0)); + + // Time 50: exactly at third mod's min, should return 2 + assertEquals(2, binarySearchMods(mods, 50, 0)); + } + + @Test + public void testBinarySearchModsWithMaxBoundaries() { + // Create mods: [10, 20], [30, 40], [50, 60] + List mods = createTestMods(new long[][] {{10, 20}, {30, 40}, {50, 60}}); + + // Time 20: exactly at first mod's max, should return 0 + assertEquals(0, binarySearchMods(mods, 20, 0)); + + // Time 40: exactly at second mod's max, should return 1 + assertEquals(1, binarySearchMods(mods, 40, 0)); + + // Time 60: exactly at third mod's max, should return 2 + assertEquals(2, binarySearchMods(mods, 60, 0)); + } + + @Test + public void testBinarySearchModsWithJustBeforeMin() { + // Create mods: [10, 20], [30, 40], [50, 60] + List mods = createTestMods(new long[][] {{10, 20}, {30, 40}, {50, 60}}); + + // Time 9: just before first mod's min, should return 0 (first mod) + assertEquals(0, binarySearchMods(mods, 9, 0)); + + // Time 29: just before second mod's min, should return 1 (second mod) + assertEquals(1, binarySearchMods(mods, 29, 0)); + + // Time 49: just before third mod's min, should return 2 (third mod) + assertEquals(2, binarySearchMods(mods, 49, 0)); + } + + @Test + public void testBinarySearchModsWithJustAfterMax() { + // Create mods: [10, 20], [30, 40], [50, 60] + List mods = createTestMods(new long[][] {{10, 20}, {30, 40}, {50, 60}}); + + // Time 21: just after first mod's max, should return 1 (second mod) + assertEquals(1, binarySearchMods(mods, 21, 0)); + + // Time 41: just after second mod's max, should return 2 (third mod) + assertEquals(2, binarySearchMods(mods, 41, 0)); + + // Time 61: just after third mod's max, should return 3 (mods.size()) + assertEquals(3, binarySearchMods(mods, 61, 0)); + } + + @Test + public void testBinarySearchModsWithLargeGaps() { + // Create mods with large gaps: [10, 20], [100, 200], [1000, 2000] + List mods = createTestMods(new long[][] {{10, 20}, {100, 200}, {1000, 2000}}); + + // Time 50: in large gap, should return 1 (second mod) + assertEquals(1, binarySearchMods(mods, 50, 0)); + + // Time 500: in large gap, should return 2 (third mod) + assertEquals(2, binarySearchMods(mods, 500, 0)); + + // Time 5000: after all mods, should return 3 (mods.size()) + assertEquals(3, binarySearchMods(mods, 5000, 0)); + } + + @Test + public void testBinarySearchModsWithNegativeTime() { + // Create mods: [10, 20], [30, 40] + List mods = createTestMods(new long[][] {{10, 20}, {30, 40}}); + + // Time -10: negative time, should return 0 (first mod) + assertEquals(0, binarySearchMods(mods, -10, 0)); + + // Time 0: zero time, should return 0 (first mod) + assertEquals(0, binarySearchMods(mods, 0, 0)); + } + + @Test + public void testBinarySearchModsWithVeryLargeTime() { + // Create mods: [10, 20], [30, 40] + List mods = createTestMods(new long[][] {{10, 20}, {30, 40}}); + + // Time Long.MAX_VALUE: very large time, should return 2 (mods.size()) + assertEquals(2, binarySearchMods(mods, Long.MAX_VALUE, 0)); + + // Time Long.MIN_VALUE: very small time, should return 0 (first mod) + assertEquals(0, binarySearchMods(mods, Long.MIN_VALUE, 0)); + } + + @Test + public void testBinarySearchModsWithDuplicateTimeRanges() { + // Create mods with duplicate time ranges: [10, 20], [10, 20], [30, 40] + List mods = createTestMods(new long[][] {{10, 20}, {10, 20}, {30, 40}}); + + // Time 15: within duplicate ranges, should return 0 (first match) + assertEquals(0, binarySearchMods(mods, 15, 0)); + + // Time 20: at max of duplicate ranges, should return 0 (first match) + assertEquals(0, binarySearchMods(mods, 20, 0)); + } + + @Test + public void testBinarySearchModsWithStartIndexAtEnd() { + // Create mods: [10, 20], [30, 40], [50, 60] + List mods = createTestMods(new long[][] {{10, 20}, {30, 40}, {50, 60}}); + + // Start from index 3 (beyond array), should return 3 + assertEquals(3, binarySearchMods(mods, 100, 3)); + + // Start from index 2, search for time 55, should return 2 + assertEquals(2, binarySearchMods(mods, 55, 2)); + + // Start from index 2, search for time 25, should return 2 (start index) + assertEquals(2, binarySearchMods(mods, 25, 2)); + } + + // Helper method to access the private binarySearchMods method for testing + private int binarySearchMods(List mods, long time, int startIndex) { + // Use reflection to access the private method + try { + java.lang.reflect.Method method = + ModsOperationUtil.class.getDeclaredMethod( + "binarySearchMods", List.class, long.class, int.class); + method.setAccessible(true); + return (Integer) method.invoke(null, mods, time, startIndex); + } catch (Exception e) { + throw new RuntimeException("Failed to invoke binarySearchMods method", e); + } + } + + private List createTestMods(long[][] timeRanges) { + List mods = new ArrayList<>(); + for (long[] range : timeRanges) { + TreeDeletionEntry mod = new TreeDeletionEntry(); + // Use reflection to set the timeRange field since it's protected + try { + java.lang.reflect.Field timeRangeField = ModEntry.class.getDeclaredField("timeRange"); + timeRangeField.setAccessible(true); + timeRangeField.set(mod, new TimeRange(range[0], range[1])); + } catch (Exception e) { + throw new RuntimeException("Failed to set timeRange", e); + } + mods.add(mod); + } + return mods; + } +} From 52111e26c046641eebc39f6330aaa25b546bc8e2 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 15 Oct 2025 15:08:38 +0800 Subject: [PATCH 19/31] modify isDelete function --- ...oTDBPipeTsFileDecompositionWithModsIT.java | 362 +++++++++++++++++- .../tsfile/parser/util/ModsOperationUtil.java | 14 +- 2 files changed, 354 insertions(+), 22 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index df1508d1a6eb..21f04efc8a52 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -39,6 +39,7 @@ public class IoTDBPipeTsFileDecompositionWithModsIT extends AbstractPipeDualTree @Override protected void setupConfig() { super.setupConfig(); + senderEnv.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true); receiverEnv.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true); } @@ -130,6 +131,22 @@ public void testTsFileDecompositionWithMods() throws Exception { TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + // Verify sender data integrity before creating pipe to avoid leader election issues + // This ensures all data is properly persisted and consistent on sender side + // before starting the pipe synchronization process + HashSet results = new HashSet<>(); + results.add("1,3.0,1.0,2.0,"); + results.add("2,3.1,null,2.1,"); + results.add("3,3.2,null,2.2,"); + results.add("4,3.3,null,2.3,"); + results.add("5,3.4,1.4,2.4,"); + + TestUtils.assertDataEventuallyOnEnv( + senderEnv, + "SELECT * FROM root.sg1.d1 ORDER BY time", + "Time,root.sg1.d1.s3,root.sg1.d1.s1,root.sg1.d1.s2,", + results); + executeNonQueryWithRetry( senderEnv, String.format( @@ -137,12 +154,6 @@ public void testTsFileDecompositionWithMods() throws Exception { receiverEnv.getDataNodeWrapperList().get(0).getIp(), receiverEnv.getDataNodeWrapperList().get(0).getPort())); - HashSet results = new HashSet<>(); - results.add("1,3.0,1.0,2.0,"); - results.add("2,3.1,null,2.1,"); - results.add("3,3.2,null,2.2,"); - results.add("4,3.3,null,2.3,"); - results.add("5,3.4,1.4,2.4,"); TestUtils.assertDataEventuallyOnEnv( receiverEnv, @@ -263,6 +274,23 @@ public void testTsFileDecompositionWithMods2() throws Exception { TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + // Verify sender data integrity before creating pipes to avoid leader election issues + // This ensures all data is properly persisted and consistent on sender side + // before starting the pipe synchronization process + HashSet results = new HashSet<>(); + results.add("1,3.0,1.0,2.0,"); + results.add("2,3.1,null,2.1,"); + results.add("3,3.2,null,2.2,"); + results.add("4,3.3,null,2.3,"); + results.add("5,3.4,1.4,2.4,"); + + TestUtils.assertDataEventuallyOnEnv( + senderEnv, + "SELECT * FROM root.sg1.d1 ORDER BY time", + "Time,root.sg1.d1.s3,root.sg1.d1.s1,root.sg1.d1.s2,", + results); + + executeNonQueryWithRetry( senderEnv, String.format( @@ -291,13 +319,6 @@ public void testTsFileDecompositionWithMods2() throws Exception { receiverEnv.getDataNodeWrapperList().get(0).getIp(), receiverEnv.getDataNodeWrapperList().get(0).getPort())); - HashSet results = new HashSet<>(); - results.add("1,3.0,1.0,2.0,"); - results.add("2,3.1,null,2.1,"); - results.add("3,3.2,null,2.2,"); - results.add("4,3.3,null,2.3,"); - results.add("5,3.4,1.4,2.4,"); - TestUtils.assertDataEventuallyOnEnv( receiverEnv, "SELECT * FROM root.sg1.d1 ORDER BY time", @@ -322,4 +343,319 @@ public void testTsFileDecompositionWithMods2() throws Exception { "COUNT(root.sg1.d4.s3),COUNT(root.sg1.d4.s1),COUNT(root.sg1.d4.s2),", Collections.singleton("3000,1000,1000,")); } + + /** + * Test IoTDB pipe handling TsFile decomposition with Mods (modification operations) in tree model + * - Large scale single point deletion scenario + * + *

Test scenario: 1. Create database root.sg1 with 1 device: d1 (aligned timeseries with 5 + * sensors) 2. Insert 20000 data points for each sensor with different time ranges: - s1: time + * 1-20000 - s2: time 10001-30000 - s3: time 20001-40000 - s4: time 30001-50000 - s5: time + * 40001-60000 3. Execute FLUSH operation to persist data to TsFile 4. Execute 2000 single point + * DELETE operations, each deleting one time point from different sensors 5. Execute FLUSH + * operation again 6. Create pipe with mods enabled 7. Verify correctness of receiver data: - Each + * sensor should have 19800 remaining data points - Deleted points should not appear in receiver + * + *

Test purpose: Verify that IoTDB pipe can correctly handle large scale single point deletion + * operations in TsFile under tree model, ensuring the binary search optimization in + * ModsOperationUtil works correctly with many modification entries. + */ + @Test + public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletion() throws Exception { + TestUtils.executeNonQueryWithRetry(senderEnv, "CREATE DATABASE root.sg1"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "CREATE DATABASE root.sg1"); + + // Insert 20000 data points for s1 (time 1-20000) + String s1 = "INSERT INTO root.sg1.d1(time, s1) ALIGNED VALUES "; + StringBuilder insertBuilder1 = new StringBuilder(s1); + for (int i = 1; i <= 20000; i++) { + insertBuilder1.append("(").append(i).append(",").append(1.0f).append(")"); + if (i % 1000 != 0) { + insertBuilder1.append(","); + } else { + TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder1.toString()); + insertBuilder1 = new StringBuilder(s1); + } + } + // Execute remaining data if any + if (insertBuilder1.length() > s1.length()) { + TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder1.toString()); + } + + // Insert 20000 data points for s2 (time 10001-30000) + String s2 = "INSERT INTO root.sg1.d1(time, s2) ALIGNED VALUES "; + StringBuilder insertBuilder2 = new StringBuilder(s2); + for (int i = 10001; i <= 30000; i++) { + insertBuilder2.append("(").append(i).append(",").append(2.0f).append(")"); + if (i % 1000 != 0) { + insertBuilder2.append(","); + } else { + TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder2.toString()); + insertBuilder2 = new StringBuilder(s2); + } + } + // Execute remaining data if any + if (insertBuilder2.length() > s2.length()) { + TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder2.toString()); + } + + // Insert 20000 data points for s3 (time 20001-40000) + String s3 = "INSERT INTO root.sg1.d1(time, s3) ALIGNED VALUES "; + StringBuilder insertBuilder3 = new StringBuilder(s3); + for (int i = 20001; i <= 40000; i++) { + insertBuilder3.append("(").append(i).append(",").append(3.0f).append(")"); + if (i % 1000 != 0) { + insertBuilder3.append(","); + } else { + TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder3.toString()); + insertBuilder3 = new StringBuilder(s3); + } + } + // Execute remaining data if any + if (insertBuilder3.length() > s3.length()) { + TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder3.toString()); + } + + // Insert 20000 data points for s4 (time 30001-50000) + String s4 = "INSERT INTO root.sg1.d1(time, s4) ALIGNED VALUES "; + StringBuilder insertBuilder4 = new StringBuilder(s4); + for (int i = 30001; i <= 50000; i++) { + insertBuilder4.append("(").append(i).append(",").append(4.0f).append(")"); + if (i % 1000 != 0) { + insertBuilder4.append(","); + } else { + TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder4.toString()); + insertBuilder4 = new StringBuilder(s4); + } + } + // Execute remaining data if any + if (insertBuilder4.length() > s4.length()) { + TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder4.toString()); + } + + // Insert 20000 data points for s5 (time 40001-60000) + String s5 = "INSERT INTO root.sg1.d1(time, s5) ALIGNED VALUES "; + StringBuilder insertBuilder5 = new StringBuilder(s5); + for (int i = 40001; i <= 60000; i++) { + insertBuilder5.append("(").append(i).append(",").append(5.0f).append(")"); + if (i % 1000 != 0) { + insertBuilder5.append(","); + } else { + TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder5.toString()); + insertBuilder5 = new StringBuilder(s5); + } + } + // Execute remaining data if any + if (insertBuilder5.length() > s5.length()) { + TestUtils.executeNonQueryWithRetry(senderEnv, insertBuilder5.toString()); + } + + TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + + // Execute 2000 single point DELETE operations + // Delete 400 points from each sensor (distributed across different time ranges) + for (int i = 0; i < 400; i++) { + // Delete from s1: every 10th point starting from 10 + TestUtils.executeNonQueryWithRetry( + senderEnv, "DELETE FROM root.sg1.d1.s1 WHERE time = " + (10 + i * 10)); + + // Delete from s2: every 10th point starting from 10010 + TestUtils.executeNonQueryWithRetry( + senderEnv, "DELETE FROM root.sg1.d1.s2 WHERE time = " + (10010 + i * 10)); + + // Delete from s3: every 10th point starting from 20010 + TestUtils.executeNonQueryWithRetry( + senderEnv, "DELETE FROM root.sg1.d1.s3 WHERE time = " + (20010 + i * 10)); + + // Delete from s4: every 10th point starting from 30010 + TestUtils.executeNonQueryWithRetry( + senderEnv, "DELETE FROM root.sg1.d1.s4 WHERE time = " + (30010 + i * 10)); + + // Delete from s5: every 10th point starting from 40010 + TestUtils.executeNonQueryWithRetry( + senderEnv, "DELETE FROM root.sg1.d1.s5 WHERE time = " + (40010 + i * 10)); + } + + TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + + // Verify sender data integrity before creating pipe to avoid leader election issues + // This ensures all data is properly persisted and consistent on sender side + // before starting the pipe synchronization process + TestUtils.assertDataEventuallyOnEnv( + senderEnv, + "SELECT COUNT(**) FROM root.sg1.d1", + "COUNT(root.sg1.d1.s3),COUNT(root.sg1.d1.s4),COUNT(root.sg1.d1.s5),COUNT(root.sg1.d1.s1),COUNT(root.sg1.d1.s2),", + Collections.singleton("19600,19600,19600,19600,19600,")); + + executeNonQueryWithRetry( + senderEnv, + String.format( + "CREATE PIPE test_pipe WITH SOURCE ('mods.enable'='true') WITH CONNECTOR('ip'='%s', 'port'='%s', 'format'='tablet')", + receiverEnv.getDataNodeWrapperList().get(0).getIp(), + receiverEnv.getDataNodeWrapperList().get(0).getPort())); + + // Verify total count of all sensors using COUNT(*) + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(**) FROM root.sg1.d1", + "COUNT(root.sg1.d1.s3),COUNT(root.sg1.d1.s4),COUNT(root.sg1.d1.s5),COUNT(root.sg1.d1.s1),COUNT(root.sg1.d1.s2),", + Collections.singleton("19600,19600,19600,19600,19600,")); + + // Verify individual sensor counts + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s1) FROM root.sg1.d1", + "COUNT(root.sg1.d1.s1),", + Collections.singleton("19600,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s2) FROM root.sg1.d1", + "COUNT(root.sg1.d1.s2),", + Collections.singleton("19600,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s3) FROM root.sg1.d1", + "COUNT(root.sg1.d1.s3),", + Collections.singleton("19600,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s4) FROM root.sg1.d1", + "COUNT(root.sg1.d1.s4),", + Collections.singleton("19600,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s5) FROM root.sg1.d1", + "COUNT(root.sg1.d1.s5),", + Collections.singleton("19600,")); + + // Verify count of deleted time ranges using COUNT with WHERE clause + // These should return 0 since all points in these ranges were deleted + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s1) FROM root.sg1.d1 WHERE time >= 10 AND time <= 4000 AND time % 10 = 0", + "COUNT(root.sg1.d1.s1),", + Collections.singleton("0,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s2) FROM root.sg1.d1 WHERE time >= 10010 AND time <= 14000 AND time % 10 = 0", + "COUNT(root.sg1.d1.s2),", + Collections.singleton("0,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s3) FROM root.sg1.d1 WHERE time >= 20010 AND time <= 24000 AND time % 10 = 0", + "COUNT(root.sg1.d1.s3),", + Collections.singleton("0,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s4) FROM root.sg1.d1 WHERE time >= 30010 AND time <= 34000 AND time % 10 = 0", + "COUNT(root.sg1.d1.s4),", + Collections.singleton("0,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s5) FROM root.sg1.d1 WHERE time >= 40010 AND time <= 44000 AND time % 10 = 0", + "COUNT(root.sg1.d1.s5),", + Collections.singleton("0,")); + + // Verify count of non-deleted time ranges using multiple range queries + // Check ranges before deletion area + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s1) FROM root.sg1.d1 WHERE time >= 1 AND time < 10", + "COUNT(root.sg1.d1.s1),", + Collections.singleton("9,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s2) FROM root.sg1.d1 WHERE time >= 10001 AND time < 10010", + "COUNT(root.sg1.d1.s2),", + Collections.singleton("9,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s3) FROM root.sg1.d1 WHERE time >= 20001 AND time < 20010", + "COUNT(root.sg1.d1.s3),", + Collections.singleton("9,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s4) FROM root.sg1.d1 WHERE time >= 30001 AND time < 30010", + "COUNT(root.sg1.d1.s4),", + Collections.singleton("9,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s5) FROM root.sg1.d1 WHERE time >= 40001 AND time < 40010", + "COUNT(root.sg1.d1.s5),", + Collections.singleton("9,")); + + // Check ranges after deletion area + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s1) FROM root.sg1.d1 WHERE time > 4000 AND time <= 20000", + "COUNT(root.sg1.d1.s1),", + Collections.singleton("16000,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s2) FROM root.sg1.d1 WHERE time > 14000 AND time <= 30000", + "COUNT(root.sg1.d1.s2),", + Collections.singleton("16000,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s3) FROM root.sg1.d1 WHERE time > 24000 AND time <= 40000", + "COUNT(root.sg1.d1.s3),", + Collections.singleton("16000,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s4) FROM root.sg1.d1 WHERE time > 34000 AND time <= 50000", + "COUNT(root.sg1.d1.s4),", + Collections.singleton("16000,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s5) FROM root.sg1.d1 WHERE time > 44000 AND time <= 60000", + "COUNT(root.sg1.d1.s5),", + Collections.singleton("16000,")); + + // Check non-deleted points within deletion range (every 10th point except the ones we deleted) + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s1) FROM root.sg1.d1 WHERE time >= 10 AND time <= 4000 AND time % 10 != 0", + "COUNT(root.sg1.d1.s1),", + Collections.singleton("3591,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s2) FROM root.sg1.d1 WHERE time >= 10010 AND time <= 14000 AND time % 10 != 0", + "COUNT(root.sg1.d1.s2),", + Collections.singleton("3591,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s3) FROM root.sg1.d1 WHERE time >= 20010 AND time <= 24000 AND time % 10 != 0", + "COUNT(root.sg1.d1.s3),", + Collections.singleton("3591,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s4) FROM root.sg1.d1 WHERE time >= 30010 AND time <= 34000 AND time % 10 != 0", + "COUNT(root.sg1.d1.s4),", + Collections.singleton("3591,")); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "SELECT COUNT(s5) FROM root.sg1.d1 WHERE time >= 40010 AND time <= 44000 AND time % 10 != 0", + "COUNT(root.sg1.d1.s5),", + Collections.singleton("3591,")); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java index b0063fb0cc59..6118038d845d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java @@ -183,8 +183,8 @@ public static boolean isDelete(long time, ModsInfo modsInfo) { final long currentModEndTime = currentMod.getTimeRange().getMax(); if (time < currentModStartTime) { - // Time is before current mod, need to search backwards - return searchAndCheckMod(mods, time, 0, modsInfo); + // Time is before current mod, return false + return false; } else if (time <= currentModEndTime) { // Time is within current mod range, return true return true; @@ -219,18 +219,14 @@ private static boolean searchAndCheckMod( final ModEntry foundMod = mods.get(searchIndex); final long foundModStartTime = foundMod.getTimeRange().getMin(); - final long foundModEndTime = foundMod.getTimeRange().getMax(); if (time < foundModStartTime) { modsInfo.setCurrentIndex(searchIndex); return false; - } else if (time <= foundModEndTime) { - modsInfo.setCurrentIndex(searchIndex); - return true; - } else { - modsInfo.setCurrentIndex(searchIndex + 1); - return false; } + + modsInfo.setCurrentIndex(searchIndex); + return true; } /** From e95bec700d9bb8f194ed9c531c0308752904ac10 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 15 Oct 2025 15:15:11 +0800 Subject: [PATCH 20/31] update --- .../IoTDBPipeTsFileDecompositionWithModsIT.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index 5a3ae45a81f7..3b8d2a72a85a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -94,6 +94,14 @@ public void testTsFileDecompositionWithMods() { "sg2", "table"); + executeNonQueryWithRetry( + senderEnv, + "DELETE FROM table1 WHERE time >= 0 AND time <= 11000 AND s0 ='t11' AND s1='t11' AND s2='t11' AND s3='t11'", + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + "sg2", + "table"); + executeNonQueryWithRetry( senderEnv, "DELETE FROM table1 WHERE time >= 5000 AND time < 10100 AND s0 ='t12' AND s1='t12' AND s2='t12' AND s3='t12'", @@ -159,13 +167,6 @@ public void testTsFileDecompositionWithMods() { Collections.singleton("0,"), "sg2"); - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t11' AND s1='t11' AND s2='t11' AND s3='t11'", - "count,", - Collections.singleton("0,"), - "sg2"); - TestUtils.assertDataEventuallyOnEnv( receiverEnv, "SELECT COUNT(*) as count FROM table1 WHERE s0 ='t12' AND s1='t12' AND s2='t12' AND s3='t12'", From 5eb48b11138cebc748a3554e16556a6a8e1f9e95 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 15 Oct 2025 15:16:16 +0800 Subject: [PATCH 21/31] spotless --- ...oTDBPipeTsFileDecompositionWithModsIT.java | 12 ++++----- ...oTDBPipeTsFileDecompositionWithModsIT.java | 26 +++++++++---------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index 3b8d2a72a85a..905a6a205a9b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -95,12 +95,12 @@ public void testTsFileDecompositionWithMods() { "table"); executeNonQueryWithRetry( - senderEnv, - "DELETE FROM table1 WHERE time >= 0 AND time <= 11000 AND s0 ='t11' AND s1='t11' AND s2='t11' AND s3='t11'", - SessionConfig.DEFAULT_USER, - SessionConfig.DEFAULT_PASSWORD, - "sg2", - "table"); + senderEnv, + "DELETE FROM table1 WHERE time >= 0 AND time <= 11000 AND s0 ='t11' AND s1='t11' AND s2='t11' AND s3='t11'", + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + "sg2", + "table"); executeNonQueryWithRetry( senderEnv, diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index 21f04efc8a52..484a450f1ff3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -142,10 +142,10 @@ public void testTsFileDecompositionWithMods() throws Exception { results.add("5,3.4,1.4,2.4,"); TestUtils.assertDataEventuallyOnEnv( - senderEnv, - "SELECT * FROM root.sg1.d1 ORDER BY time", - "Time,root.sg1.d1.s3,root.sg1.d1.s1,root.sg1.d1.s2,", - results); + senderEnv, + "SELECT * FROM root.sg1.d1 ORDER BY time", + "Time,root.sg1.d1.s3,root.sg1.d1.s1,root.sg1.d1.s2,", + results); executeNonQueryWithRetry( senderEnv, @@ -154,7 +154,6 @@ public void testTsFileDecompositionWithMods() throws Exception { receiverEnv.getDataNodeWrapperList().get(0).getIp(), receiverEnv.getDataNodeWrapperList().get(0).getPort())); - TestUtils.assertDataEventuallyOnEnv( receiverEnv, "SELECT * FROM root.sg1.d1 ORDER BY time", @@ -285,11 +284,10 @@ public void testTsFileDecompositionWithMods2() throws Exception { results.add("5,3.4,1.4,2.4,"); TestUtils.assertDataEventuallyOnEnv( - senderEnv, - "SELECT * FROM root.sg1.d1 ORDER BY time", - "Time,root.sg1.d1.s3,root.sg1.d1.s1,root.sg1.d1.s2,", - results); - + senderEnv, + "SELECT * FROM root.sg1.d1 ORDER BY time", + "Time,root.sg1.d1.s3,root.sg1.d1.s1,root.sg1.d1.s2,", + results); executeNonQueryWithRetry( senderEnv, @@ -482,10 +480,10 @@ public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletion() throw // This ensures all data is properly persisted and consistent on sender side // before starting the pipe synchronization process TestUtils.assertDataEventuallyOnEnv( - senderEnv, - "SELECT COUNT(**) FROM root.sg1.d1", - "COUNT(root.sg1.d1.s3),COUNT(root.sg1.d1.s4),COUNT(root.sg1.d1.s5),COUNT(root.sg1.d1.s1),COUNT(root.sg1.d1.s2),", - Collections.singleton("19600,19600,19600,19600,19600,")); + senderEnv, + "SELECT COUNT(**) FROM root.sg1.d1", + "COUNT(root.sg1.d1.s3),COUNT(root.sg1.d1.s4),COUNT(root.sg1.d1.s5),COUNT(root.sg1.d1.s1),COUNT(root.sg1.d1.s2),", + Collections.singleton("19600,19600,19600,19600,19600,")); executeNonQueryWithRetry( senderEnv, From 4ac2699c8f1328217f5f708e3c4771ef5fccbe7a Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 15 Oct 2025 16:33:45 +0800 Subject: [PATCH 22/31] update IT Config --- .../manual/IoTDBPipeTsFileDecompositionWithModsIT.java | 6 ++++++ .../manual/IoTDBPipeTsFileDecompositionWithModsIT.java | 1 + 2 files changed, 7 insertions(+) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index 905a6a205a9b..93c71548639d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -38,6 +38,12 @@ @Category({MultiClusterIT2DualTableManualBasic.class}) public class IoTDBPipeTsFileDecompositionWithModsIT extends AbstractPipeTableModelDualManualIT { + @Override + protected void setupConfig() { + super.setupConfig(); + senderEnv.getConfig().getCommonConfig().setEnableAutoLeaderBalanceForIoTConsensus(false); + } + /** * Test IoTDB pipe handling TsFile decomposition with Mods (modification operations) in table * model diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index 484a450f1ff3..dc4795903726 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -39,6 +39,7 @@ public class IoTDBPipeTsFileDecompositionWithModsIT extends AbstractPipeDualTree @Override protected void setupConfig() { super.setupConfig(); + senderEnv.getConfig().getCommonConfig().setEnableAutoLeaderBalanceForIoTConsensus(false); senderEnv.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true); receiverEnv.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true); } From b06c0ad918319610ea8d2f100591ffb79df3c74e Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Wed, 15 Oct 2025 18:34:44 +0800 Subject: [PATCH 23/31] fix --- .../manual/IoTDBPipeTsFileDecompositionWithModsIT.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index dc4795903726..e52e4fcc3ebb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -39,7 +39,6 @@ public class IoTDBPipeTsFileDecompositionWithModsIT extends AbstractPipeDualTree @Override protected void setupConfig() { super.setupConfig(); - senderEnv.getConfig().getCommonConfig().setEnableAutoLeaderBalanceForIoTConsensus(false); senderEnv.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true); receiverEnv.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true); } @@ -132,6 +131,9 @@ public void testTsFileDecompositionWithMods() throws Exception { TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + Thread.sleep( + 30000); // wait for followers to catch up, avoiding leader election during pipe creation + // Verify sender data integrity before creating pipe to avoid leader election issues // This ensures all data is properly persisted and consistent on sender side // before starting the pipe synchronization process @@ -274,6 +276,9 @@ public void testTsFileDecompositionWithMods2() throws Exception { TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + Thread.sleep( + 30000); // wait for followers to catch up, avoiding leader election during pipe creation + // Verify sender data integrity before creating pipes to avoid leader election issues // This ensures all data is properly persisted and consistent on sender side // before starting the pipe synchronization process @@ -477,6 +482,9 @@ public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletion() throw TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); + Thread.sleep( + 30000); // wait for followers to catch up, avoiding leader election during pipe creation + // Verify sender data integrity before creating pipe to avoid leader election issues // This ensures all data is properly persisted and consistent on sender side // before starting the pipe synchronization process From 42586637d88f91efe79f3a559d98216e116598ab Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Thu, 16 Oct 2025 09:49:31 +0800 Subject: [PATCH 24/31] update IoTDBPipeTsFileDecompositionWithModsIT --- .../IoTDBPipeTsFileDecompositionWithModsIT.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/{ => basic}/IoTDBPipeTsFileDecompositionWithModsIT.java (96%) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeTsFileDecompositionWithModsIT.java similarity index 96% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeTsFileDecompositionWithModsIT.java index 93c71548639d..22e042e28538 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -17,13 +17,14 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.isession.SessionConfig; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -38,12 +39,6 @@ @Category({MultiClusterIT2DualTableManualBasic.class}) public class IoTDBPipeTsFileDecompositionWithModsIT extends AbstractPipeTableModelDualManualIT { - @Override - protected void setupConfig() { - super.setupConfig(); - senderEnv.getConfig().getCommonConfig().setEnableAutoLeaderBalanceForIoTConsensus(false); - } - /** * Test IoTDB pipe handling TsFile decomposition with Mods (modification operations) in table * model From 5876a46477f98965aaa1ae6980c212bbc6edfb3f Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Thu, 16 Oct 2025 17:46:05 +0800 Subject: [PATCH 25/31] update IoTDBPipeTsFileDecompositionWithModsIT --- .../manual/IoTDBPipeTsFileDecompositionWithModsIT.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java index e52e4fcc3ebb..b939ec79b8be 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTsFileDecompositionWithModsIT.java @@ -39,6 +39,7 @@ public class IoTDBPipeTsFileDecompositionWithModsIT extends AbstractPipeDualTree @Override protected void setupConfig() { super.setupConfig(); + senderEnv.getConfig().getConfigNodeConfig().setLeaderDistributionPolicy("HASH"); senderEnv.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true); receiverEnv.getConfig().getCommonConfig().setAutoCreateSchemaEnabled(true); } @@ -131,9 +132,6 @@ public void testTsFileDecompositionWithMods() throws Exception { TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); - Thread.sleep( - 30000); // wait for followers to catch up, avoiding leader election during pipe creation - // Verify sender data integrity before creating pipe to avoid leader election issues // This ensures all data is properly persisted and consistent on sender side // before starting the pipe synchronization process @@ -276,9 +274,6 @@ public void testTsFileDecompositionWithMods2() throws Exception { TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); - Thread.sleep( - 30000); // wait for followers to catch up, avoiding leader election during pipe creation - // Verify sender data integrity before creating pipes to avoid leader election issues // This ensures all data is properly persisted and consistent on sender side // before starting the pipe synchronization process @@ -482,9 +477,6 @@ public void testTsFileDecompositionWithModsLargeScaleSinglePointDeletion() throw TestUtils.executeNonQueryWithRetry(senderEnv, "FLUSH"); - Thread.sleep( - 30000); // wait for followers to catch up, avoiding leader election during pipe creation - // Verify sender data integrity before creating pipe to avoid leader election issues // This ensures all data is properly persisted and consistent on sender side // before starting the pipe synchronization process From efcbf122cef363846090f6017487b4f2443adcdb Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Thu, 16 Oct 2025 17:58:12 +0800 Subject: [PATCH 26/31] update --- .../tsfile/parser/util/ModsOperationUtilTest.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtilTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtilTest.java index d0a4037fb90d..7eb09bbab41a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtilTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtilTest.java @@ -400,15 +400,7 @@ private int binarySearchMods(List mods, long time, int startIndex) { private List createTestMods(long[][] timeRanges) { List mods = new ArrayList<>(); for (long[] range : timeRanges) { - TreeDeletionEntry mod = new TreeDeletionEntry(); - // Use reflection to set the timeRange field since it's protected - try { - java.lang.reflect.Field timeRangeField = ModEntry.class.getDeclaredField("timeRange"); - timeRangeField.setAccessible(true); - timeRangeField.set(mod, new TimeRange(range[0], range[1])); - } catch (Exception e) { - throw new RuntimeException("Failed to set timeRange", e); - } + TreeDeletionEntry mod = new TreeDeletionEntry(null, new TimeRange(range[0], range[1])); mods.add(mod); } return mods; From 4b7cf343735a890cd5c5d3d93986451b175e623d Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Thu, 23 Oct 2025 12:31:58 +0800 Subject: [PATCH 27/31] fix --- .../TsFileInsertionEventQueryParser.java | 55 ++++++++-------- ...sertionEventQueryParserTabletIterator.java | 6 +- .../scan/TsFileInsertionEventScanParser.java | 66 ++++++++++--------- ...sertionEventTableParserTabletIterator.java | 17 +++-- 4 files changed, 72 insertions(+), 72 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java index db0e105dcb5e..115a952c61ff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java @@ -194,32 +194,35 @@ public TsFileInsertionEventQueryParser( continue; } - // Safely filter measurements, remove non-existent measurements - measurements.removeIf( - measurement -> { - if (measurement == null || measurement.isEmpty()) { - LOGGER.warn("Found null or empty measurement for deviceId: {}, removing", deviceId); - return true; - } - - try { - TimeseriesMetadata meta = - tsFileSequenceReader.readTimeseriesMetadata(deviceId, measurement, true); - return ModsOperationUtil.isAllDeletedByMods( - deviceId, - measurement, - meta.getStatistics().getStartTime(), - meta.getStatistics().getEndTime(), - currentModifications); - } catch (IOException e) { - LOGGER.warn( - "Failed to read metadata for deviceId: {}, measurement: {}, removing", - deviceId, - measurement, - e); - return true; - } - }); + if (!currentModifications.isEmpty()) { + // Safely filter measurements, remove non-existent measurements + measurements.removeIf( + measurement -> { + if (measurement == null || measurement.isEmpty()) { + LOGGER.warn( + "Found null or empty measurement for deviceId: {}, removing", deviceId); + return true; + } + + try { + TimeseriesMetadata meta = + tsFileSequenceReader.readTimeseriesMetadata(deviceId, measurement, true); + return ModsOperationUtil.isAllDeletedByMods( + deviceId, + measurement, + meta.getStatistics().getStartTime(), + meta.getStatistics().getEndTime(), + currentModifications); + } catch (IOException e) { + LOGGER.warn( + "Failed to read metadata for deviceId: {}, measurement: {}, removing", + deviceId, + measurement, + e); + return true; + } + }); + } // If measurements list is empty after filtering, remove the entire entry if (measurements.isEmpty()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java index ceb63ac133bb..776b5e1e6fac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java @@ -176,19 +176,15 @@ private Tablet buildNextTablet() throws IOException { final int rowIndex = tablet.getRowSize(); boolean isNeedFillTime = false; - final List fields = rowRecord.getFields(); final int fieldSize = fields.size(); for (int i = 0; i < fieldSize; i++) { final Field field = fields.get(i); final String measurement = measurements.get(i); - boolean isDeletedByMods = false; // Check if this value is deleted by mods if (field == null - || (isDeletedByMods = - ModsOperationUtil.isDelete(rowRecord.getTimestamp(), measurementModsList.get(i)))) { + || ModsOperationUtil.isDelete(rowRecord.getTimestamp(), measurementModsList.get(i))) { tablet.getBitMaps()[i].mark(rowIndex); - isNeedFillTime = isNeedFillTime || !isDeletedByMods; } else { tablet.addValue(measurement, rowIndex, field.getObjectValue(schemas.get(i).getType())); isNeedFillTime = true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java index b78975f3ee06..19f0acae05f1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java @@ -339,9 +339,8 @@ private boolean putValueToColumns(final BatchData data, final Tablet tablet, fin if (data.getDataType() == TSDataType.VECTOR) { for (int i = 0; i < tablet.getSchemas().size(); ++i) { final TsPrimitiveType primitiveType = data.getVector()[i]; - boolean isDeleted = false; if (Objects.isNull(primitiveType) - || (isDeleted = ModsOperationUtil.isDelete(data.currentTime(), modsInfos.get(i)))) { + || ModsOperationUtil.isDelete(data.currentTime(), modsInfos.get(i))) { switch (tablet.getSchemas().get(i).getType()) { case TEXT: case BLOB: @@ -349,7 +348,6 @@ private boolean putValueToColumns(final BatchData data, final Tablet tablet, fin tablet.addValue(rowIndex, i, Binary.EMPTY_VALUE.getValues()); } tablet.getBitMaps()[i].mark(rowIndex); - isNeedFillTime = isNeedFillTime || !isDeleted; continue; } @@ -468,20 +466,22 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { } // Skip the chunk if it is fully deleted by mods - final Statistics statistics = - findNonAlignedChunkStatistics( - tsFileSequenceReader.getIChunkMetadataList( - currentDevice, chunkHeader.getMeasurementID()), - currentChunkHeaderOffset); - if (statistics != null - && ModsOperationUtil.isAllDeletedByMods( - currentDevice, - chunkHeader.getMeasurementID(), - statistics.getStartTime(), - statistics.getEndTime(), - currentModifications)) { - tsFileSequenceReader.position(nextMarkerOffset); - break; + if (!currentModifications.isEmpty()) { + final Statistics statistics = + findNonAlignedChunkStatistics( + tsFileSequenceReader.getIChunkMetadataList( + currentDevice, chunkHeader.getMeasurementID()), + currentChunkHeaderOffset); + if (statistics != null + && ModsOperationUtil.isAllDeletedByMods( + currentDevice, + chunkHeader.getMeasurementID(), + statistics.getStartTime(), + statistics.getEndTime(), + currentModifications)) { + tsFileSequenceReader.position(nextMarkerOffset); + break; + } } if (chunkHeader.getDataSize() > allocatedMemoryBlockForChunk.getMemoryUsageInBytes()) { @@ -523,21 +523,23 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { break; } - // Skip the chunk if it is fully deleted by mods - final Statistics statistics = - findAlignedChunkStatistics( - tsFileSequenceReader.getIChunkMetadataList( - currentDevice, chunkHeader.getMeasurementID()), - currentChunkHeaderOffset); - if (statistics != null - && ModsOperationUtil.isAllDeletedByMods( - currentDevice, - chunkHeader.getMeasurementID(), - statistics.getStartTime(), - statistics.getEndTime(), - currentModifications)) { - tsFileSequenceReader.position(nextMarkerOffset); - break; + if (!currentModifications.isEmpty()) { + // Skip the chunk if it is fully deleted by mods + final Statistics statistics = + findAlignedChunkStatistics( + tsFileSequenceReader.getIChunkMetadataList( + currentDevice, chunkHeader.getMeasurementID()), + currentChunkHeaderOffset); + if (statistics != null + && ModsOperationUtil.isAllDeletedByMods( + currentDevice, + chunkHeader.getMeasurementID(), + statistics.getStartTime(), + statistics.getEndTime(), + currentModifications)) { + tsFileSequenceReader.position(nextMarkerOffset); + break; + } } // Increase value index diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java index ce2f4fe353d3..86dc4b12bbda 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java @@ -221,12 +221,13 @@ public boolean hasNext() { continue; } - if (ModsOperationUtil.isAllDeletedByMods( - pair.getLeft(), - iChunkMetadata.getMeasurementUid(), - alignedChunkMetadata.getStartTime(), - alignedChunkMetadata.getEndTime(), - modifications)) { + if (!modifications.isEmpty() + && ModsOperationUtil.isAllDeletedByMods( + pair.getLeft(), + iChunkMetadata.getMeasurementUid(), + alignedChunkMetadata.getStartTime(), + alignedChunkMetadata.getEndTime(), + modifications)) { iChunkMetadataIterator.remove(); } } @@ -432,9 +433,8 @@ private boolean fillMeasurementValueColumns( for (int i = deviceIdSize, size = dataTypeList.size(); i < size; i++) { final TsPrimitiveType primitiveType = primitiveTypes[i - deviceIdSize]; - boolean isDelete = false; if (primitiveType == null - || (isDelete = ModsOperationUtil.isDelete(data.currentTime(), modsInfoList.get(i)))) { + || ModsOperationUtil.isDelete(data.currentTime(), modsInfoList.get(i))) { switch (dataTypeList.get(i)) { case TEXT: case BLOB: @@ -442,7 +442,6 @@ private boolean fillMeasurementValueColumns( tablet.addValue(rowIndex, i, Binary.EMPTY_VALUE.getValues()); } tablet.getBitMaps()[i].mark(rowIndex); - needFillTime = needFillTime || !isDelete; continue; } needFillTime = true; From 81a04b99d7dfafb20c1dc276d30ed00ae86ced9f Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Thu, 23 Oct 2025 14:11:56 +0800 Subject: [PATCH 28/31] update ModsOperationUtil --- .../common/tsfile/parser/util/ModsOperationUtil.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java index 6118038d845d..66fed43feade 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/util/ModsOperationUtil.java @@ -134,10 +134,10 @@ public static List initializeMeasurementMods( // Sort by time range for efficient lookup // Different filtering logic for tree model and table model - final List sortedMods; + final List filteredMods; if (deviceID.isTableModel()) { // For table model: filter modifications that affect the device - sortedMods = + filteredMods = mods.stream() .filter( modification -> @@ -145,10 +145,10 @@ public static List initializeMeasurementMods( .collect(Collectors.toList()); } else { // For tree model: no additional filtering needed - sortedMods = mods; + filteredMods = mods; } // Store sorted mods and start index - modsInfos.add(new ModsInfo(ModificationUtils.sortAndMerge(sortedMods), 0)); + modsInfos.add(new ModsInfo(ModificationUtils.sortAndMerge(filteredMods), 0)); } return modsInfos; From 967f8406cb4051c8d5df09a8e8a0e483336119ff Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Thu, 23 Oct 2025 15:05:23 +0800 Subject: [PATCH 29/31] update TsFileInsertionEventTableParserTabletIterator --- .../table/TsFileInsertionEventTableParserTabletIterator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java index 86dc4b12bbda..f05cf872c798 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/table/TsFileInsertionEventTableParserTabletIterator.java @@ -218,6 +218,7 @@ public boolean hasNext() { while (iChunkMetadataIterator.hasNext()) { IChunkMetadata iChunkMetadata = iChunkMetadataIterator.next(); if (iChunkMetadata == null) { + iChunkMetadataIterator.remove(); continue; } From 256966a9e0351ec9d9dbe19dc46d5cb7f32ded5e Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Thu, 23 Oct 2025 15:26:19 +0800 Subject: [PATCH 30/31] update --- .../parser/query/TsFileInsertionEventQueryParser.java | 6 +----- .../tsfile/parser/scan/TsFileInsertionEventScanParser.java | 6 +++++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java index 115a952c61ff..8905e7d58e7e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java @@ -188,8 +188,6 @@ public TsFileInsertionEventQueryParser( // Check if measurements list is deleted or empty if (measurements == null || measurements.isEmpty()) { - LOGGER.warn( - "Found null or empty measurements list for deviceId: {}, removing entry", deviceId); iterator.remove(); continue; } @@ -198,9 +196,7 @@ public TsFileInsertionEventQueryParser( // Safely filter measurements, remove non-existent measurements measurements.removeIf( measurement -> { - if (measurement == null || measurement.isEmpty()) { - LOGGER.warn( - "Found null or empty measurement for deviceId: {}, removing", deviceId); + if (measurement == null) { return true; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java index 19f0acae05f1..0ccac56be11c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/scan/TsFileInsertionEventScanParser.java @@ -130,7 +130,11 @@ public TsFileInsertionEventScanParser( PipeDataNodeResourceManager.memory() .forceAllocateForTabletWithRetry(currentModifications.ramBytesUsed()); - tsFileSequenceReader = new TsFileSequenceReader(tsFile.getAbsolutePath(), true, true); + tsFileSequenceReader = + new TsFileSequenceReader( + tsFile.getAbsolutePath(), + !currentModifications.isEmpty(), + !currentModifications.isEmpty()); tsFileSequenceReader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1); prepareData(); From 1082ef97128b17b8355bad45e21f760f90cff794 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Thu, 23 Oct 2025 15:33:28 +0800 Subject: [PATCH 31/31] update --- .../tsfile/parser/query/TsFileInsertionEventQueryParser.java | 1 - 1 file changed, 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java index 8905e7d58e7e..dbed85d5a6b9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java @@ -222,7 +222,6 @@ public TsFileInsertionEventQueryParser( // If measurements list is empty after filtering, remove the entire entry if (measurements.isEmpty()) { - LOGGER.warn("No valid measurements left for deviceId: {}, removing entry", deviceId); iterator.remove(); } }