diff --git a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBQueryWithIDTableIT.java b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBQueryWithIDTableIT.java index 6980b7cd4774..4c8e2a027a84 100644 --- a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBQueryWithIDTableIT.java +++ b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBQueryWithIDTableIT.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.integration; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.deviceID.DeviceIDFactory; import org.apache.iotdb.integration.env.ConfigFactory; import org.apache.iotdb.integration.env.EnvFactory; import org.apache.iotdb.itbase.category.LocalStandaloneTest; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java index 60e0fecefa5b..5645eddd7350 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java @@ -52,7 +52,7 @@ import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.exception.runtime.StorageEngineFailureException; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.deviceID.DeviceIDFactory; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java index a3ca350fea24..35b29e7e19af 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java @@ -25,7 +25,7 @@ import org.apache.iotdb.db.engine.memtable.IWritableMemChunk; import org.apache.iotdb.db.engine.memtable.IWritableMemChunkGroup; import org.apache.iotdb.db.exception.runtime.FlushRunTimeException; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.rescon.SystemInfo; import org.apache.iotdb.db.service.metrics.MetricService; import org.apache.iotdb.db.service.metrics.enums.Metric; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java index 91c3375fc1b1..46194be4c11f 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java @@ -26,8 +26,8 @@ import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.metadata.utils.ResourceByPathUtils; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java index adf995e9487b..8e54667e0add 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java @@ -25,7 +25,7 @@ import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java index 9bcf4f5807d7..30951af8622d 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.engine.memtable; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import java.util.HashMap; import java.util.Map; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java index 9d1bac5d9953..75977afeac75 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java @@ -44,8 +44,8 @@ import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.exception.WriteProcessRejectException; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.metadata.path.AlignedPath; import org.apache.iotdb.db.metadata.utils.ResourceByPathUtils; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowNode; diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/AppendOnlyDiskSchemaManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/AppendOnlyDiskSchemaManager.java index 785644479144..67c8249f7c23 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/AppendOnlyDiskSchemaManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/AppendOnlyDiskSchemaManager.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.deviceID.DeviceIDFactory; import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; @@ -148,7 +148,6 @@ public void recover(IDTable idTable) { try (FileInputStream inputStream = new FileInputStream(dataFile)) { // read file version ReadWriteIOUtils.readString(inputStream); - while (inputStream.available() > 0) { DiskSchemaEntry cur = DiskSchemaEntry.deserialize(inputStream); if (!cur.deviceID.equals(DiskSchemaEntry.TOMBSTONE)) { @@ -158,7 +157,12 @@ public void recover(IDTable idTable) { TSEncoding.deserialize(cur.encoding), CompressionType.deserialize(cur.compressor), loc); - idTable.putSchemaEntry(cur.deviceID, cur.measurementName, schemaEntry, cur.isAligned); + idTable.putSchemaEntry( + cur.deviceID, + getDevicePathFromSeriesKey(cur.seriesKey, cur.measurementName), + cur.measurementName, + schemaEntry, + cur.isAligned); } loc += cur.entrySize; } @@ -248,7 +252,7 @@ private DiskSchemaEntry getDiskSchemaEntryByOffset(long offset) throws IOExcepti String measurementName = readString(); String deviceID = DeviceIDFactory.getInstance() - .getDeviceID(seriesKey.substring(0, seriesKey.length() - measurementName.length() - 1)) + .getDeviceID(getDevicePathFromSeriesKey(seriesKey, measurementName)) .toStringID(); return new DiskSchemaEntry( deviceID, @@ -267,6 +271,10 @@ private String readString() throws IOException { return new String(bytes, 0, strLength); } + private String getDevicePathFromSeriesKey(String seriesKey, String measurement) { + return seriesKey.substring(0, seriesKey.length() - measurement.length() - 1); + } + @Override public void close() throws IOException { try { diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTable.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTable.java index fa44865e4e75..ecd7800bcd31 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTable.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTable.java @@ -24,10 +24,10 @@ import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.idtable.deviceID.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; import org.apache.iotdb.db.metadata.idtable.entry.TimeseriesID; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; @@ -133,6 +133,7 @@ void updateLastCache( throws MetadataException; /** clear id table and close file */ + @TestOnly void clear() throws IOException; /** @@ -143,6 +144,14 @@ void updateLastCache( */ DeviceEntry getDeviceEntry(String deviceName); + /** + * get device entry from device id + * + * @param deviceID device id of the device path + * @return device entry + */ + DeviceEntry getDeviceEntry(IDeviceID deviceID); + /** * get schema from device and measurements * @@ -162,13 +171,19 @@ void updateLastCache( /** * put schema entry to id table, currently used in recover * - * @param devicePath device path (can be device id formed path) + * @param deviceID device id + * @param devicePath device path * @param measurement measurement name * @param schemaEntry schema entry to put * @param isAligned is the device aligned + * @throws MetadataException */ void putSchemaEntry( - String devicePath, String measurement, SchemaEntry schemaEntry, boolean isAligned) + String deviceID, + String devicePath, + String measurement, + SchemaEntry schemaEntry, + boolean isAligned) throws MetadataException; /** diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableAutoIncImpl.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableAutoIncImpl.java new file mode 100644 index 000000000000..e081485f8a87 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableAutoIncImpl.java @@ -0,0 +1,319 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.idtable; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.localconfignode.LocalConfigNode; +import org.apache.iotdb.db.metadata.idtable.deviceID.AutoIncDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.SHA256DeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.metadata.idtable.entry.TimeseriesID; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class IDTableAutoIncImpl extends IDTableHashmapImpl { + + /** logger */ + private static final Logger logger = LoggerFactory.getLogger(IDTableAutoIncImpl.class); + + /** config */ + protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + // if the schemaRegionId==-1 of a AutoIncDeviceID instance, it means that the device corresponding + // to the AutoIncDeviceID instance does not exist return the deviceIdOfNonExistentDevice object + // for all devicePaths of unmanaged devices in the metadata module, which can avoid unnecessary + // object creation while ensuring correctness + private static AutoIncDeviceID deviceIdOfNonExistentDevice; + + // stand-alone auto-increment id uses LocalConfigNode to obtain schemaRegionId + private static LocalConfigNode configManager; + + // using map to maintain the mapping from schemaRegionId to list, each list + // maintains the auto-increment id of the schemaRegion + private static Map> deviceEntriesMap; + + static { + deviceEntriesMap = new ConcurrentHashMap<>(); + configManager = LocalConfigNode.getInstance(); + deviceIdOfNonExistentDevice = new AutoIncDeviceID(-1, 0); + } + + /** + * get a AutoIncDeviceID instance, if it does not exist, return deviceIdOfNonExistentDevice object + * + * @param deviceID device id or device path + * @return a AutoIncDeviceID instance + */ + public static AutoIncDeviceID getDeviceID(String deviceID) { + if (deviceID.startsWith("`") && deviceID.endsWith("`")) { + return getDeviceIdFromDeviceEntriesMap(deviceID); + } else { + return getDeviceIdFromIdTable(deviceID); + } + } + + /** + * get the AutoIncDeviceID instance from deviceEntriesMap by device id + * + * @param deviceID device id, like:"`1`" + * @return a AutoIncDeviceID instance + */ + private static AutoIncDeviceID getDeviceIdFromDeviceEntriesMap(String deviceID) { + AutoIncDeviceID autoIncID = new AutoIncDeviceID(deviceID); + int schemaRegionId = autoIncID.getSchemaRegionId(); + int autoIncrementID = autoIncID.getAutoIncrementID(); + if (schemaRegionId == -1) { + return deviceIdOfNonExistentDevice; + } + List deviceEntries = deviceEntriesMap.get(schemaRegionId); + synchronized (deviceEntries) { + DeviceEntry deviceEntry = deviceEntries.get(autoIncrementID); + return (AutoIncDeviceID) deviceEntry.getDeviceID(); + } + } + + /** + * get the AutoIncDeviceID instance from IdTable by device path + * + * @param devicePath device path, like:"root.sg.x.d1" + * @return a AutoIncDeviceID instance + */ + private static AutoIncDeviceID getDeviceIdFromIdTable(String devicePath) { + try { + // use idTable to determine whether the device has been created + IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath(devicePath)); + SHA256DeviceID sha256DeviceID = new SHA256DeviceID(devicePath); + DeviceEntry deviceEntry = idTable.getDeviceEntry(sha256DeviceID); + if (idTable.getDeviceEntry(sha256DeviceID) != null) { + return (AutoIncDeviceID) deviceEntry.getDeviceID(); + } else { + return deviceIdOfNonExistentDevice; + } + } catch (IllegalPathException e) { + logger.info(e.getMessage()); + return deviceIdOfNonExistentDevice; + } + } + + public IDTableAutoIncImpl(File storageGroupDir) { + super(storageGroupDir); + } + + /** + * used in insert operation + * + * @param deviceName device name of the time series + * @param isAligned whether the insert plan is aligned + * @return a device entry + * @throws MetadataException + */ + @Override + protected DeviceEntry getDeviceEntryWithAlignedCheck(String deviceName, boolean isAligned) + throws MetadataException { + DeviceEntry deviceEntry = null; + if (deviceName.startsWith("`") && deviceName.endsWith("`")) { + return getDeviceEntryFromDeviceEntriesMap(deviceName); + } else { + SHA256DeviceID sha256DeviceID = new SHA256DeviceID(deviceName); + int slot = calculateSlot(sha256DeviceID); + Map[] idtables = getIdTables(); + deviceEntry = idtables[slot].get(sha256DeviceID); + + // new device + if (deviceEntry == null) { + deviceEntry = getDeviceEntryWIthAutoCreate(deviceName); + deviceEntry.setAligned(isAligned); + idtables[slot].put(sha256DeviceID, deviceEntry); + } + } + // check aligned + if (deviceEntry.isAligned() != isAligned) { + throw new MetadataException( + String.format( + "Timeseries under path [%s]'s align value is [%b], which is not consistent with insert plan", + deviceName, deviceEntry.isAligned())); + } + + // reuse device entry in map + return deviceEntry; + } + + /** + * put schema entry to id table, currently used in recover + * + * @param deviceID device id + * @param devicePath device path + * @param measurement measurement name + * @param schemaEntry schema entry to put + * @param isAligned is the device aligned + */ + @Override + public void putSchemaEntry( + String deviceID, + String devicePath, + String measurement, + SchemaEntry schemaEntry, + boolean isAligned) { + AutoIncDeviceID autoIncDeviceID = new AutoIncDeviceID(deviceID); + int schemaRegionID = autoIncDeviceID.getSchemaRegionId(); + int autoIncrementID = autoIncDeviceID.getAutoIncrementID(); + + SHA256DeviceID sha256DeviceID = new SHA256DeviceID(devicePath); + DeviceEntry deviceEntry = getDeviceEntry(sha256DeviceID); + if (deviceEntry == null) { + deviceEntry = new DeviceEntry(autoIncDeviceID); + List deviceEntries = + deviceEntriesMap.computeIfAbsent(schemaRegionID, integer -> new ArrayList<>()); + // deviceEntriesMap recover + synchronized (deviceEntries) { + // handle out-of-order + if (autoIncrementID < deviceEntries.size()) { + deviceEntries.set(autoIncrementID, deviceEntry); + } else { + for (int i = deviceEntries.size(); i < autoIncrementID; i++) { + deviceEntries.add(i, null); + } + deviceEntries.add(autoIncrementID, deviceEntry); + } + } + // idTable recover + int slot = calculateSlot(sha256DeviceID); + Map[] idtables = getIdTables(); + idtables[slot].put(sha256DeviceID, deviceEntry); + deviceEntry.setAligned(isAligned); + } + deviceEntry.putSchemaEntry(measurement, schemaEntry); + } + + /** + * get Device entry + * + * @param deviceName device name of the time series + * @return device entry + */ + @Override + public DeviceEntry getDeviceEntry(String deviceName) { + if (deviceName.startsWith("`") && deviceName.endsWith("`")) { + return getDeviceEntryFromDeviceEntriesMap(deviceName); + } else { + SHA256DeviceID sha256DeviceID = new SHA256DeviceID(deviceName); + return getDeviceEntry(sha256DeviceID); + } + } + + /** + * get schemaEntry + * + * @param timeseriesID the timeseries ID + * @return schema entry + * @throws MetadataException + */ + @Override + protected SchemaEntry getSchemaEntry(TimeseriesID timeseriesID) throws MetadataException { + AutoIncDeviceID deviceID = (AutoIncDeviceID) timeseriesID.getDeviceID(); + List deviceEntries = deviceEntriesMap.get(deviceID.getSchemaRegionId()); + if (deviceEntries == null) { + throw new MetadataException( + "get non exist timeseries's schema entry, timeseries id is: " + timeseriesID); + } + synchronized (deviceEntries) { + DeviceEntry deviceEntry = deviceEntries.get(deviceID.getAutoIncrementID()); + if (deviceEntry == null) { + throw new MetadataException( + "get non exist timeseries's schema entry, timeseries id is: " + timeseriesID); + } + + SchemaEntry schemaEntry = deviceEntry.getSchemaEntry(timeseriesID.getMeasurement()); + if (schemaEntry == null) { + throw new MetadataException( + "get non exist timeseries's schema entry, timeseries id is: " + timeseriesID); + } + return schemaEntry; + } + } + + /** + * create a deviceEntry object in deviceEntriesMap and return the instance, which holds the newly + * created AutoIncDeviceID instance + * + * @param devicePath device path, like: "root.sg.x.d1" + * @return a deviceEntry which holds the newly created AutoIncDeviceID instance + * @throws MetadataException + */ + private DeviceEntry getDeviceEntryWIthAutoCreate(String devicePath) throws MetadataException { + int schemaRegionID = + configManager.getBelongedSchemaRegionId(new PartialPath(devicePath)).getId(); + List deviceEntries = + deviceEntriesMap.computeIfAbsent(schemaRegionID, integer -> new ArrayList<>()); + synchronized (deviceEntries) { + int autoIncrementID = deviceEntries.size(); + AutoIncDeviceID deviceID = new AutoIncDeviceID(schemaRegionID, autoIncrementID); + DeviceEntry deviceEntry = new DeviceEntry(deviceID); + deviceEntries.add(deviceEntry); + return deviceEntry; + } + } + + /** + * get DeviceEntry from deviceEntriesMap by device id + * + * @param deviceID device id, like: "`1`" + * @return device entry + */ + private DeviceEntry getDeviceEntryFromDeviceEntriesMap(String deviceID) { + AutoIncDeviceID autoIncID = new AutoIncDeviceID(deviceID); + int schemaRegionId = autoIncID.getSchemaRegionId(); + int autoIncrementID = autoIncID.getAutoIncrementID(); + List deviceEntries = deviceEntriesMap.get(schemaRegionId); + synchronized (deviceEntries) { + return deviceEntries.get(autoIncrementID); + } + } + + @Override + @TestOnly + public void clear() throws IOException { + super.clear(); + deviceEntriesMap.clear(); + configManager = null; + deviceIdOfNonExistentDevice = null; + } + + @TestOnly + public static void reset() { + deviceEntriesMap.clear(); + configManager = LocalConfigNode.getInstance(); + deviceIdOfNonExistentDevice = new AutoIncDeviceID(-1, 0); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableHashmapImpl.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableHashmapImpl.java index af3cb4d9bc03..0f638797a0c1 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableHashmapImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableHashmapImpl.java @@ -25,10 +25,10 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; +import org.apache.iotdb.db.metadata.idtable.deviceID.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.InsertMeasurementMNode; import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; import org.apache.iotdb.db.metadata.idtable.entry.TimeseriesID; @@ -72,6 +72,7 @@ public class IDTableHashmapImpl implements IDTable { /** disk schema manager to manage disk schema entry */ private IDiskSchemaManager IDiskSchemaManager; + /** iotdb config */ protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); @@ -307,6 +308,7 @@ public synchronized void updateLastCache( } @Override + @TestOnly public void clear() throws IOException { if (IDiskSchemaManager != null) { IDiskSchemaManager.close(); @@ -321,7 +323,17 @@ public void clear() throws IOException { */ @Override public DeviceEntry getDeviceEntry(String deviceName) { - IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(deviceName); + return getDeviceEntry(DeviceIDFactory.getInstance().getDeviceID(deviceName)); + } + + /** + * get device entry from deviceID + * + * @param deviceID deviceID of the device path + * @return device entry + */ + @Override + public DeviceEntry getDeviceEntry(IDeviceID deviceID) { int slot = calculateSlot(deviceID); // reuse device entry in map @@ -355,6 +367,11 @@ public IMeasurementSchema getSeriesSchema(String deviceName, String measurementN schemaEntry.getCompressionType()); } + /** + * get all useful deviceEntry of idTable + * + * @return device entries + */ @Override public List getAllDeviceEntry() { List res = new ArrayList<>(); @@ -365,9 +382,23 @@ public List getAllDeviceEntry() { return res; } + /** + * put schema entry to id table, currently used in recover + * + * @param deviceID device id + * @param devicePath device path + * @param measurement measurement name + * @param schemaEntry schema entry to put + * @param isAligned is the device aligned + * @throws MetadataException + */ @Override public void putSchemaEntry( - String devicePath, String measurement, SchemaEntry schemaEntry, boolean isAligned) + String deviceID, + String devicePath, + String measurement, + SchemaEntry schemaEntry, + boolean isAligned) throws MetadataException { DeviceEntry deviceEntry = getDeviceEntryWithAlignedCheck(devicePath, isAligned); deviceEntry.putSchemaEntry(measurement, schemaEntry); @@ -452,13 +483,13 @@ private IMeasurementMNode getOrCreateMeasurementIfNotExist( } /** - * get device id from device path and check is aligned, + * get device entry from device path and check is aligned * * @param deviceName device name of the time series * @param isAligned whether the insert plan is aligned * @return device entry of the timeseries */ - private DeviceEntry getDeviceEntryWithAlignedCheck(String deviceName, boolean isAligned) + protected DeviceEntry getDeviceEntryWithAlignedCheck(String deviceName, boolean isAligned) throws MetadataException { IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(deviceName); int slot = calculateSlot(deviceID); @@ -469,10 +500,7 @@ private DeviceEntry getDeviceEntryWithAlignedCheck(String deviceName, boolean is deviceEntry = new DeviceEntry(deviceID); deviceEntry.setAligned(isAligned); idTables[slot].put(deviceID, deviceEntry); - - return deviceEntry; } - // check aligned if (deviceEntry.isAligned() != isAligned) { throw new MetadataException( @@ -491,7 +519,7 @@ private DeviceEntry getDeviceEntryWithAlignedCheck(String deviceName, boolean is * @param deviceID device id * @return slot number */ - private int calculateSlot(IDeviceID deviceID) { + protected int calculateSlot(IDeviceID deviceID) { int hashVal = deviceID.hashCode(); return Math.abs(hashVal == Integer.MIN_VALUE ? 0 : hashVal) % NUM_OF_SLOTS; } @@ -503,7 +531,7 @@ private int calculateSlot(IDeviceID deviceID) { * @return schema entry of the timeseries * @throws MetadataException throw if this timeseries is not exist */ - private SchemaEntry getSchemaEntry(TimeseriesID timeseriesID) throws MetadataException { + protected SchemaEntry getSchemaEntry(TimeseriesID timeseriesID) throws MetadataException { IDeviceID deviceID = timeseriesID.getDeviceID(); int slot = calculateSlot(deviceID); @@ -523,7 +551,6 @@ private SchemaEntry getSchemaEntry(TimeseriesID timeseriesID) throws MetadataExc } @Override - @TestOnly public Map[] getIdTables() { return idTables; } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableManager.java index f60e18812aa1..49692ef431b2 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableManager.java @@ -21,8 +21,10 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.metadata.idtable.deviceID.DeviceIDMode; import org.apache.iotdb.db.service.IoTDB; import org.apache.iotdb.tsfile.utils.FilePathUtils; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; @@ -82,10 +84,19 @@ public synchronized IDTable getIDTable(PartialPath devicePath) { try { return idTableMap.computeIfAbsent( IoTDB.schemaProcessor.getStorageGroupNodeByPath(devicePath).getFullPath(), - storageGroupPath -> - new IDTableHashmapImpl( - SystemFileFactory.INSTANCE.getFile( - systemDir + File.separator + storageGroupPath))); + storageGroupPath -> { + switch (DeviceIDMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod())) { + case AutoIncrement: + return new IDTableAutoIncImpl( + SystemFileFactory.INSTANCE.getFile( + systemDir + File.separator + storageGroupPath)); + default: + return new IDTableHashmapImpl( + SystemFileFactory.INSTANCE.getFile( + systemDir + File.separator + storageGroupPath)); + } + }); } catch (MetadataException e) { logger.error("get id table failed, path is: " + devicePath + ". caused by: " + e); } @@ -102,9 +113,19 @@ public synchronized IDTable getIDTable(PartialPath devicePath) { public synchronized IDTable getIDTableDirectly(String sgPath) { return idTableMap.computeIfAbsent( sgPath, - storageGroupPath -> - new IDTableHashmapImpl( - SystemFileFactory.INSTANCE.getFile(systemDir + File.separator + storageGroupPath))); + storageGroupPath -> { + switch (DeviceIDMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod())) { + case AutoIncrement: + return new IDTableAutoIncImpl( + SystemFileFactory.INSTANCE.getFile( + systemDir + File.separator + storageGroupPath)); + default: + return new IDTableHashmapImpl( + SystemFileFactory.INSTANCE.getFile( + systemDir + File.separator + storageGroupPath)); + } + }); } /** @@ -127,11 +148,11 @@ public synchronized IMeasurementSchema getSeriesSchema(String deviceName, String } /** clear id table map */ + @TestOnly public void clear() throws IOException { for (IDTable idTable : idTableMap.values()) { idTable.clear(); } - idTableMap.clear(); } } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/AutoIncDeviceID.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/AutoIncDeviceID.java new file mode 100644 index 000000000000..cd23553be8c9 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/AutoIncDeviceID.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.idtable.deviceID; + +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; + +import java.nio.ByteBuffer; + +public class AutoIncDeviceID implements IDeviceID { + + // starting with 0,the maximum value is Integer.MAX_VALUE + int schemaRegionId; + + // starting with 0,the maximum value is Integer.MAX_VALUE + int autoIncrementID; + + public AutoIncDeviceID() {} + + public AutoIncDeviceID(String deviceID) { + long id = parseFromDeviceID(deviceID); + this.schemaRegionId = (int) (id >>> 32); + this.autoIncrementID = (int) id; + } + + public AutoIncDeviceID(int schemaRegionId, int autoIncrementID) { + this.schemaRegionId = schemaRegionId; + this.autoIncrementID = autoIncrementID; + } + + private long parseFromDeviceID(String deviceID) { + deviceID = deviceID.substring(1, deviceID.length() - 1); + return Long.parseLong(deviceID); + } + + @Override + public int hashCode() { + long id = (long) schemaRegionId << 32; + id |= autoIncrementID; + return Long.hashCode(id); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof AutoIncDeviceID)) { + return false; + } + AutoIncDeviceID that = (AutoIncDeviceID) o; + return this.autoIncrementID == that.autoIncrementID + && this.schemaRegionId == that.schemaRegionId; + } + + @Override + public String toString() { + return "AutoIncDeviceID{" + + "schemaRegionId=" + + schemaRegionId + + ", autoIncrementID=" + + autoIncrementID + + '}'; + } + + @Override + public String toStringID() { + long stringID = (long) schemaRegionId << 32; + stringID |= autoIncrementID; + return "`" + stringID + '`'; + } + + @Override + public void serialize(ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(schemaRegionId, byteBuffer); + ReadWriteIOUtils.write(autoIncrementID, byteBuffer); + } + + public static AutoIncDeviceID deserialize(ByteBuffer byteBuffer) { + AutoIncDeviceID deviceID = new AutoIncDeviceID(); + deviceID.schemaRegionId = ReadWriteIOUtils.readInt(byteBuffer); + deviceID.autoIncrementID = ReadWriteIOUtils.readInt(byteBuffer); + return deviceID; + } + + public int getSchemaRegionId() { + return schemaRegionId; + } + + public int getAutoIncrementID() { + return autoIncrementID; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDFactory.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/DeviceIDFactory.java similarity index 64% rename from server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDFactory.java rename to server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/DeviceIDFactory.java index 332183cb3bec..c1973817ad9d 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDFactory.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/DeviceIDFactory.java @@ -17,11 +17,12 @@ * under the License. */ -package org.apache.iotdb.db.metadata.idtable.entry; +package org.apache.iotdb.db.metadata.idtable.deviceID; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.idtable.IDTableAutoIncImpl; import java.util.function.Function; @@ -49,15 +50,21 @@ public static DeviceIDFactory getInstance() { } private DeviceIDFactory() { - if (IoTDBDescriptor.getInstance().getConfig().isEnableIDTable() - && IoTDBDescriptor.getInstance() - .getConfig() - .getDeviceIDTransformationMethod() - .equals("SHA256")) { - getDeviceIDFunction = SHA256DeviceID::new; - } else { - getDeviceIDFunction = PlainDeviceID::new; + if (IoTDBDescriptor.getInstance().getConfig().isEnableIDTable()) { + switch (DeviceIDMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod())) { + case SHA256: + getDeviceIDFunction = SHA256DeviceID::new; + return; + case AutoIncrement: + getDeviceIDFunction = IDTableAutoIncImpl::getDeviceID; + return; + default: + getDeviceIDFunction = PlainDeviceID::new; + return; + } } + getDeviceIDFunction = PlainDeviceID::new; } // endregion @@ -65,7 +72,7 @@ private DeviceIDFactory() { * get device id by full path * * @param devicePath device path of the timeseries - * @return device id of the timeseries + * @return a IDeviceID instance of the device path */ public IDeviceID getDeviceID(PartialPath devicePath) { return getDeviceIDFunction.apply(devicePath.toString()); @@ -75,7 +82,7 @@ public IDeviceID getDeviceID(PartialPath devicePath) { * get device id by full path * * @param devicePath device path of the timeseries - * @return device id of the timeseries + * @return a IDeviceID instance of the device path */ public IDeviceID getDeviceID(String devicePath) { return getDeviceIDFunction.apply(devicePath); @@ -84,14 +91,21 @@ public IDeviceID getDeviceID(String devicePath) { /** reset id method */ @TestOnly public void reset() { - if (IoTDBDescriptor.getInstance().getConfig().isEnableIDTable() - && IoTDBDescriptor.getInstance() - .getConfig() - .getDeviceIDTransformationMethod() - .equals("SHA256")) { - getDeviceIDFunction = SHA256DeviceID::new; - } else { - getDeviceIDFunction = PlainDeviceID::new; + if (IoTDBDescriptor.getInstance().getConfig().isEnableIDTable()) { + switch (DeviceIDMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod())) { + case SHA256: + getDeviceIDFunction = SHA256DeviceID::new; + return; + case AutoIncrement: + getDeviceIDFunction = IDTableAutoIncImpl::getDeviceID; + IDTableAutoIncImpl.reset(); + return; + default: + getDeviceIDFunction = PlainDeviceID::new; + return; + } } + getDeviceIDFunction = PlainDeviceID::new; } } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/DeviceIDMode.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/DeviceIDMode.java new file mode 100644 index 000000000000..859d9d67f28d --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/DeviceIDMode.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.idtable.deviceID; + +public enum DeviceIDMode { + Plain, + SHA256, + AutoIncrement +} diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/IDeviceID.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/IDeviceID.java similarity index 88% rename from server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/IDeviceID.java rename to server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/IDeviceID.java index f83dc525a528..881d41e9d2c6 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/IDeviceID.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/IDeviceID.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.metadata.idtable.entry; +package org.apache.iotdb.db.metadata.idtable.deviceID; import java.nio.ByteBuffer; @@ -29,7 +29,7 @@ public interface IDeviceID { * * @return string format device id */ - public String toStringID(); + String toStringID(); - public void serialize(ByteBuffer byteBuffer); + void serialize(ByteBuffer byteBuffer); } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/PlainDeviceID.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/PlainDeviceID.java similarity index 97% rename from server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/PlainDeviceID.java rename to server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/PlainDeviceID.java index eac298cd7087..a799d55fd591 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/PlainDeviceID.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/PlainDeviceID.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.metadata.idtable.entry; +package org.apache.iotdb.db.metadata.idtable.deviceID; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SHA256DeviceID.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/SHA256DeviceID.java similarity index 97% rename from server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SHA256DeviceID.java rename to server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/SHA256DeviceID.java index 1042aa698573..fd88a4613b0a 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SHA256DeviceID.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/SHA256DeviceID.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.db.metadata.idtable.entry; +package org.apache.iotdb.db.metadata.idtable.deviceID; import org.apache.iotdb.db.metadata.idtable.IDTable; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; @@ -90,7 +90,7 @@ private void fromSHA256String(String deviceID) { * * @param deviceID device path */ - private void buildSHA256(String deviceID) { + protected void buildSHA256(String deviceID) { byte[] hashVal; int slot = calculateSlot(deviceID); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceEntry.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceEntry.java index 3ef77edec71b..0f42b41ab463 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceEntry.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceEntry.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.metadata.idtable.entry; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; + import java.util.HashMap; import java.util.Map; import java.util.Objects; diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SchemaEntry.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SchemaEntry.java index bf95324a4b28..589a7d198697 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SchemaEntry.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SchemaEntry.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.metadata.idtable.IDiskSchemaManager; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.metadata.lastCache.container.ILastCacheContainer; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/TimeseriesID.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/TimeseriesID.java index 143ddbc3b37e..e02b38ea20fc 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/TimeseriesID.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/TimeseriesID.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.metadata.idtable.entry; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.metadata.idtable.deviceID.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; /** A po class contains device id and measurement, represents a timeseries */ public class TimeseriesID { diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/utils/ResourceByPathUtils.java b/server/src/main/java/org/apache/iotdb/db/metadata/utils/ResourceByPathUtils.java index 74b8c0b59dc7..7902c8c2580b 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/utils/ResourceByPathUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/utils/ResourceByPathUtils.java @@ -33,8 +33,8 @@ import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.metadata.path.AlignedPath; import org.apache.iotdb.db.metadata.path.MeasurementPath; import org.apache.iotdb.db.query.context.QueryContext; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertNode.java index 610628b33434..9cfcfccccf1e 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertNode.java @@ -23,7 +23,7 @@ import org.apache.iotdb.consensus.multileader.wal.ConsensusReqReader; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.mpp.common.schematree.ISchemaTree; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.mpp.plan.planner.plan.node.WritePlanNode; diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java index a5ef81ff12a5..1333692df520 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; import org.apache.iotdb.db.qp.logical.Operator; import org.apache.iotdb.db.qp.physical.PhysicalPlan; diff --git a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/TsFilePlanRedoer.java b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/TsFilePlanRedoer.java index d69798fdcad9..16d527913e1d 100644 --- a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/TsFilePlanRedoer.java +++ b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/TsFilePlanRedoer.java @@ -31,7 +31,7 @@ import org.apache.iotdb.db.exception.metadata.PathNotExistException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.metadata.idtable.IDTable; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.deviceID.DeviceIDFactory; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; import org.apache.iotdb.db.mpp.plan.analyze.SchemaValidator; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertNode; diff --git a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/UnsealedTsFileRecoverPerformer.java index 926e54a75b22..631a6cfe7e23 100644 --- a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/UnsealedTsFileRecoverPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/UnsealedTsFileRecoverPerformer.java @@ -28,7 +28,7 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.DataRegionException; import org.apache.iotdb.db.metadata.idtable.IDTable; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.qp.physical.crud.DeletePlan; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableTestUtils.java b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableTestUtils.java index 7b0a2085a8d4..8289a2d6e37a 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableTestUtils.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemTableTestUtils.java @@ -20,7 +20,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.deviceID.DeviceIDFactory; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; import org.apache.iotdb.db.metadata.mnode.MeasurementMNode; import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemtableBenchmark.java b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemtableBenchmark.java index 04434179a9d5..a7d69c6f96c6 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemtableBenchmark.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/memtable/MemtableBenchmark.java @@ -20,7 +20,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.deviceID.DeviceIDFactory; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java b/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java index fd01cd0f9345..85c7b74ac6ae 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java @@ -25,7 +25,7 @@ import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.deviceID.DeviceIDFactory; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; import org.apache.iotdb.db.metadata.mnode.MeasurementMNode; import org.apache.iotdb.db.metadata.path.AlignedPath; diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableFlushTimeTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableFlushTimeTest.java index cb48267bbf72..ffc0ee3a96a4 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableFlushTimeTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableFlushTimeTest.java @@ -70,19 +70,19 @@ public void before() { isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement"); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile); - EnvironmentUtils.cleanEnv(); } @Test diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableLogFileTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableLogFileTest.java index eb2b337518fd..7e6029b45801 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableLogFileTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableLogFileTest.java @@ -22,7 +22,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.StorageEngineException; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.deviceID.DeviceIDFactory; import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; @@ -57,19 +57,19 @@ public void before() { isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement"); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile); - EnvironmentUtils.cleanEnv(); } @Test diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRecoverTest.java index bcacb1607358..0649550347b0 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRecoverTest.java @@ -63,32 +63,32 @@ public void before() { isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement"); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile); - - EnvironmentUtils.cleanEnv(); } @Test public void testRecover() throws Exception { - insertDataInMemoryWithTablet(false); - insertDataInMemoryWithRecord(false); - + insertDataInMemoryWithTablet("root.isp1.d2", false); + insertDataInMemoryWithRecord("root.isp1.d1", false); + insertDataInMemoryWithTablet("root.isp2.d2", false); + insertDataInMemoryWithRecord("root.isp2.d1", false); PlanExecutor executor = new PlanExecutor(); PhysicalPlan flushPlan = processor.parseSQLToPhysicalPlan("flush"); executor.processNonQuery(flushPlan); - IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp")); + IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp1")); List memoryList = idTable.getAllDeviceEntry(); // restart @@ -100,22 +100,40 @@ public void testRecover() throws Exception { // check id table fields - idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp.d1")); + idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp1.d1")); List recoverList = idTable.getAllDeviceEntry(); assertEquals(memoryList, recoverList); + + idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp2")); + memoryList = idTable.getAllDeviceEntry(); + + // restart + try { + EnvironmentUtils.restartDaemon(); + } catch (Exception e) { + Assert.fail(); + } + + // check id table fields + + idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp2.d1")); + recoverList = idTable.getAllDeviceEntry(); + + assertEquals(memoryList, recoverList); } @Test public void testRecoverAligned() throws Exception { - insertDataInMemoryWithTablet(true); - insertDataInMemoryWithRecord(false); - + insertDataInMemoryWithTablet("root.isp1.d2", true); + insertDataInMemoryWithRecord("root.isp1.d1", true); + insertDataInMemoryWithTablet("root.isp2.d2", true); + insertDataInMemoryWithRecord("root.isp2.d1", true); PlanExecutor executor = new PlanExecutor(); PhysicalPlan flushPlan = processor.parseSQLToPhysicalPlan("flush"); executor.processNonQuery(flushPlan); - IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp")); + IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp1")); List memoryList = idTable.getAllDeviceEntry(); // restart @@ -127,13 +145,30 @@ public void testRecoverAligned() throws Exception { // check id table fields - idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp.d1")); + idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp1.d1")); List recoverList = idTable.getAllDeviceEntry(); assertEquals(memoryList, recoverList); + + idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp2")); + memoryList = idTable.getAllDeviceEntry(); + + // restart + try { + EnvironmentUtils.restartDaemon(); + } catch (Exception e) { + Assert.fail(); + } + + // check id table fields + + idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp2.d1")); + recoverList = idTable.getAllDeviceEntry(); + + assertEquals(memoryList, recoverList); } - private void insertDataInMemoryWithRecord(boolean isAligned) + private void insertDataInMemoryWithRecord(String storageGroupPath, boolean isAligned) throws IllegalPathException, QueryProcessException { long time = 100L; TSDataType[] dataTypes = @@ -156,7 +191,7 @@ private void insertDataInMemoryWithRecord(boolean isAligned) InsertRowPlan insertRowPlan = new InsertRowPlan( - new PartialPath("root.isp.d1"), + new PartialPath(storageGroupPath), time, new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, dataTypes, @@ -167,7 +202,7 @@ private void insertDataInMemoryWithRecord(boolean isAligned) executor.insert(insertRowPlan); } - private void insertDataInMemoryWithTablet(boolean isAligned) + private void insertDataInMemoryWithTablet(String storageGroupPath, boolean isAligned) throws IllegalPathException, QueryProcessException { long[] times = new long[] {110L, 111L, 112L, 113L}; List dataTypes = new ArrayList<>(); @@ -197,7 +232,7 @@ private void insertDataInMemoryWithTablet(boolean isAligned) InsertTabletPlan tabletPlan = new InsertTabletPlan( - new PartialPath("root.isp.d2"), + new PartialPath(storageGroupPath), new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, dataTypes); tabletPlan.setTimes(times); diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableResourceControlTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableResourceControlTest.java index 35c9ff4f1b13..cc41dd8c8107 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableResourceControlTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableResourceControlTest.java @@ -27,8 +27,8 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileProcessor; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; import org.apache.iotdb.db.qp.Planner; import org.apache.iotdb.db.qp.executor.PlanExecutor; import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; @@ -65,17 +65,17 @@ public void before() { IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement"); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); - EnvironmentUtils.cleanEnv(); } @Test @@ -102,9 +102,9 @@ public void testDeviceIDReusing() continue; } - for (IDeviceID deviceID : map.keySet()) { + for (DeviceEntry deviceEntry : map.values()) { if (idTableDeviceID == null) { - idTableDeviceID = deviceID; + idTableDeviceID = deviceEntry.getDeviceID(); } else { fail("there should only be one device in id table"); } diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRestartTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRestartTest.java index f4801710df50..a6eecd58c735 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRestartTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRestartTest.java @@ -66,26 +66,51 @@ public void before() { isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement"); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile); - - EnvironmentUtils.cleanEnv(); } @Test public void testRawDataQueryAfterRestart() throws Exception { - insertDataInMemoryWithTablet(); - insertDataInMemoryWithRecord(); + String path1 = "root.isp1.d1"; + String path2 = "root.isp2.d1"; + insertDataInMemoryWithTablet(path1); + insertDataInMemoryWithTablet(path2); + insertDataInMemoryWithRecord(path1); + insertDataInMemoryWithRecord(path2); + + PlanExecutor executor = new PlanExecutor(); + QueryPlan queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from " + path1); + QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + Assert.assertEquals(6, dataSet.getPaths().size()); + int count = 0; + while (dataSet.hasNext()) { + RowRecord record = dataSet.next(); + System.out.println(record); + count++; + } + + executor = new PlanExecutor(); + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from " + path2); + dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + Assert.assertEquals(6, dataSet.getPaths().size()); + count = 0; + while (dataSet.hasNext()) { + RowRecord record = dataSet.next(); + System.out.println(record); + count++; + } // restart try { @@ -94,11 +119,24 @@ public void testRawDataQueryAfterRestart() throws Exception { Assert.fail(); } - PlanExecutor executor = new PlanExecutor(); - QueryPlan queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from root.isp.d1"); - QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + executor = new PlanExecutor(); + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from " + path1); + dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); Assert.assertEquals(6, dataSet.getPaths().size()); - int count = 0; + count = 0; + while (dataSet.hasNext()) { + RowRecord record = dataSet.next(); + System.out.println(record); + count++; + } + + assertEquals(5, count); + + executor = new PlanExecutor(); + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from " + path2); + dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + Assert.assertEquals(6, dataSet.getPaths().size()); + count = 0; while (dataSet.hasNext()) { RowRecord record = dataSet.next(); System.out.println(record); @@ -116,9 +154,21 @@ public void testRawDataQueryAfterRestart() throws Exception { } catch (Exception e) { Assert.fail(); } + executor = new PlanExecutor(); + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from " + path1); + dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + Assert.assertEquals(6, dataSet.getPaths().size()); + count = 0; + while (dataSet.hasNext()) { + RowRecord record = dataSet.next(); + System.out.println(record); + count++; + } + + assertEquals(5, count); executor = new PlanExecutor(); - queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from root.isp.d1"); + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from " + path2); dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); Assert.assertEquals(6, dataSet.getPaths().size()); count = 0; @@ -131,7 +181,8 @@ public void testRawDataQueryAfterRestart() throws Exception { assertEquals(5, count); } - private void insertDataInMemoryWithRecord() throws IllegalPathException, QueryProcessException { + private void insertDataInMemoryWithRecord(String path) + throws IllegalPathException, QueryProcessException { long time = 100L; TSDataType[] dataTypes = new TSDataType[] { @@ -149,11 +200,11 @@ private void insertDataInMemoryWithRecord() throws IllegalPathException, QueryPr columns[2] = 10000 + ""; columns[3] = 100 + ""; columns[4] = false + ""; - columns[5] = "hh" + 0; + columns[5] = path; InsertRowPlan insertRowPlan = new InsertRowPlan( - new PartialPath("root.isp.d1"), + new PartialPath(path), time, new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, dataTypes, @@ -163,7 +214,8 @@ private void insertDataInMemoryWithRecord() throws IllegalPathException, QueryPr executor.insert(insertRowPlan); } - private void insertDataInMemoryWithTablet() throws IllegalPathException, QueryProcessException { + private void insertDataInMemoryWithTablet(String path) + throws IllegalPathException, QueryProcessException { long[] times = new long[] {110L, 111L, 112L, 113L}; List dataTypes = new ArrayList<>(); dataTypes.add(TSDataType.DOUBLE.ordinal()); @@ -187,14 +239,12 @@ private void insertDataInMemoryWithTablet() throws IllegalPathException, QueryPr ((long[]) columns[2])[r] = 100000 + r; ((int[]) columns[3])[r] = 1000 + r; ((boolean[]) columns[4])[r] = false; - ((Binary[]) columns[5])[r] = new Binary("mm" + r); + ((Binary[]) columns[5])[r] = new Binary(path); } InsertTabletPlan tabletPlan = new InsertTabletPlan( - new PartialPath("root.isp.d1"), - new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, - dataTypes); + new PartialPath(path), new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, dataTypes); tabletPlan.setTimes(times); tabletPlan.setColumns(columns); tabletPlan.setRowCount(times.length); diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableTest.java index 2ea9f8cd35a8..56aa5625d77f 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableTest.java @@ -27,9 +27,9 @@ import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.metadata.LocalSchemaProcessor; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.deviceID.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; import org.apache.iotdb.db.metadata.lastCache.container.ILastCacheContainer; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; @@ -88,19 +88,19 @@ public void before() { isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement"); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile); - EnvironmentUtils.cleanEnv(); } @Test @@ -617,6 +617,7 @@ public void testGetDiskSchemaEntries() { IDeviceID iDeviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath); String measurement = "s" + i; idTable.putSchemaEntry( + iDeviceID.toStringID(), devicePath, measurement, new SchemaEntry( @@ -661,7 +662,7 @@ public void testDeleteTimeseries() { new PartialPath(devicePath + "." + measurement), false, idTable.getIDiskSchemaManager()); - idTable.putSchemaEntry(devicePath, measurement, schemaEntry, false); + idTable.putSchemaEntry(iDeviceID.toStringID(), devicePath, measurement, schemaEntry, false); } List partialPaths = new ArrayList<>(); partialPaths.add(new PartialPath("root.laptop.d0.s0")); diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/InsertWithIDTableTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/InsertWithIDTableTest.java index 15f677a6a1bb..b3d8fe6d4185 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/InsertWithIDTableTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/InsertWithIDTableTest.java @@ -75,17 +75,17 @@ public void before() { IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement"); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); - EnvironmentUtils.cleanEnv(); } @Test @@ -755,5 +755,10 @@ public void testInsertMultiTabletPlan() RowRecord record = dataSet.next(); assertEquals(60, record.getFields().size()); } + + // query for records that do not exist + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from root.multi.d11"); + dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + assertEquals(0, dataSet.getPaths().size()); } } diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/LastQueryWithIDTable.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/LastQueryWithIDTable.java index 33bbd75d0a09..00a6ae978813 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/LastQueryWithIDTable.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/LastQueryWithIDTable.java @@ -65,12 +65,18 @@ public class LastQueryWithIDTable { Set retSet = new HashSet<>( Arrays.asList( - "113\troot.isp.d1.s3\t100003\tINT64", - "113\troot.isp.d1.s4\t1003\tINT32", - "113\troot.isp.d1.s5\tfalse\tBOOLEAN", - "113\troot.isp.d1.s6\tmm3\tTEXT", - "113\troot.isp.d1.s1\t13.0\tDOUBLE", - "113\troot.isp.d1.s2\t23.0\tFLOAT")); + "113\troot.isp1.d1.s3\t100003\tINT64", + "113\troot.isp1.d1.s4\t1003\tINT32", + "113\troot.isp1.d1.s5\tfalse\tBOOLEAN", + "113\troot.isp1.d1.s6\tmm3\tTEXT", + "113\troot.isp1.d1.s1\t13.0\tDOUBLE", + "113\troot.isp1.d1.s2\t23.0\tFLOAT", + "113\troot.isp2.d1.s3\t100003\tINT64", + "113\troot.isp2.d1.s4\t1003\tINT32", + "113\troot.isp2.d1.s5\tfalse\tBOOLEAN", + "113\troot.isp2.d1.s6\tmm3\tTEXT", + "113\troot.isp2.d1.s1\t13.0\tDOUBLE", + "113\troot.isp2.d1.s2\t23.0\tFLOAT")); @Before public void before() { @@ -82,18 +88,18 @@ public void before() { IoTDBDescriptor.getInstance().getConfig().setEnableLastCache(false); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement"); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); IoTDBDescriptor.getInstance().getConfig().setEnableLastCache(originalEnableCache); - EnvironmentUtils.cleanEnv(); } @Test @@ -101,11 +107,12 @@ public void testLastCacheQueryWithoutCache() throws QueryProcessException, MetadataException, InterruptedException, QueryFilterOptimizationException, StorageEngineException, IOException { - insertDataInMemory(); + insertDataInMemory("root.isp1.d1"); + insertDataInMemory("root.isp2.d1"); PlanExecutor executor = new PlanExecutor(); QueryPlan queryPlan = - (QueryPlan) processor.parseSQLToPhysicalPlan("select last * from root.isp.d1"); + (QueryPlan) processor.parseSQLToPhysicalPlan("select last * from root.isp1.d1"); QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); Assert.assertEquals(3, dataSet.getPaths().size()); int count = 0; @@ -115,12 +122,26 @@ public void testLastCacheQueryWithoutCache() count++; } - assertEquals(retSet.size(), count); + assertEquals(6, count); + + executor = new PlanExecutor(); + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select last * from root.isp2.d1"); + dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + Assert.assertEquals(3, dataSet.getPaths().size()); + count = 0; + while (dataSet.hasNext()) { + RowRecord record = dataSet.next(); + assertTrue(retSet.contains(record.toString())); + count++; + } + + assertEquals(6, count); // flush and test again PhysicalPlan flushPlan = processor.parseSQLToPhysicalPlan("flush"); executor.processNonQuery(flushPlan); + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select last * from root.isp1.d1"); dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); Assert.assertEquals(3, dataSet.getPaths().size()); count = 0; @@ -129,16 +150,37 @@ public void testLastCacheQueryWithoutCache() assertTrue(retSet.contains(record.toString())); count++; } - assertEquals(retSet.size(), count); + assertEquals(6, count); // assert id table is not refresh assertNull( IDTableManager.getInstance() - .getIDTable(new PartialPath("root.isp.d1")) - .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s1")))); + .getIDTable(new PartialPath("root.isp1.d1")) + .getLastCache(new TimeseriesID(new PartialPath("root.isp1.d1.s1")))); + + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select last * from root.isp2.d1"); + dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + Assert.assertEquals(3, dataSet.getPaths().size()); + count = 0; + while (dataSet.hasNext()) { + RowRecord record = dataSet.next(); + assertTrue(retSet.contains(record.toString())); + count++; + } + assertEquals(6, count); + + // assert id table is not refresh + assertNull( + IDTableManager.getInstance() + .getIDTable(new PartialPath("root.isp2.d1")) + .getLastCache(new TimeseriesID(new PartialPath("root.isp2.d1.s1")))); + + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select last * from root.isp2.d2"); + dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + Assert.assertEquals(0, dataSet.getPaths().size()); } - private void insertDataInMemory() throws IllegalPathException, QueryProcessException { + private void insertDataInMemory(String path) throws IllegalPathException, QueryProcessException { long[] times = new long[] {110L, 111L, 112L, 113L}; List dataTypes = new ArrayList<>(); dataTypes.add(TSDataType.DOUBLE.ordinal()); @@ -167,9 +209,7 @@ private void insertDataInMemory() throws IllegalPathException, QueryProcessExcep InsertTabletPlan tabletPlan = new InsertTabletPlan( - new PartialPath("root.isp.d1"), - new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, - dataTypes); + new PartialPath(path), new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, dataTypes); tabletPlan.setTimes(times); tabletPlan.setColumns(columns); tabletPlan.setRowCount(times.length); diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/QueryAlignedTimeseriesWithIDTableTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/QueryAlignedTimeseriesWithIDTableTest.java index 22c77325b18f..b4ee28c6c53e 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/QueryAlignedTimeseriesWithIDTableTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/QueryAlignedTimeseriesWithIDTableTest.java @@ -142,17 +142,17 @@ public void before() { IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement"); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); - EnvironmentUtils.cleanEnv(); } @Test diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/QueryWithIDTableTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/QueryWithIDTableTest.java index ca44e1fc8f7f..83ecfab6ea50 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/QueryWithIDTableTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/QueryWithIDTableTest.java @@ -145,19 +145,19 @@ public void before() { isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement"); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile); - EnvironmentUtils.cleanEnv(); } @Test diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/deviceID/DeviceIDTest.java similarity index 62% rename from server/src/test/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDTest.java rename to server/src/test/java/org/apache/iotdb/db/metadata/idtable/deviceID/DeviceIDTest.java index 54ebc0afeba6..23717b36c06f 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/deviceID/DeviceIDTest.java @@ -17,17 +17,45 @@ * under the License. */ -package org.apache.iotdb.db.metadata.idtable.entry; +package org.apache.iotdb.db.metadata.idtable.deviceID; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; public class DeviceIDTest { + + private boolean isEnableIDTable = false; + + private String originalDeviceIDTransformationMethod = null; + + @Before + public void setUp() throws Exception { + isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable(); + originalDeviceIDTransformationMethod = + IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod(); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + EnvironmentUtils.envSetUp(); + } + + @After + public void tearDown() throws Exception { + EnvironmentUtils.cleanEnv(); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); + IoTDBDescriptor.getInstance() + .getConfig() + .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); + } + @Test public void deviceIDBuildTest() throws IllegalPathException { PartialPath partialPath1 = new PartialPath("root.sg1.d1.s1"); diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java index df70dcd56bae..46cafd3f4b2a 100644 --- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java +++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java @@ -39,7 +39,7 @@ import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.TriggerManagementException; import org.apache.iotdb.db.metadata.idtable.IDTableManager; -import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.deviceID.DeviceIDFactory; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.control.FileReaderManager; import org.apache.iotdb.db.query.control.QueryResourceManager;