diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java index a3ca350fea24..35b29e7e19af 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java @@ -25,7 +25,7 @@ import org.apache.iotdb.db.engine.memtable.IWritableMemChunk; import org.apache.iotdb.db.engine.memtable.IWritableMemChunkGroup; import org.apache.iotdb.db.exception.runtime.FlushRunTimeException; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.rescon.SystemInfo; import org.apache.iotdb.db.service.metrics.MetricService; import org.apache.iotdb.db.service.metrics.enums.Metric; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java index 91c3375fc1b1..d5e723875c0c 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java @@ -26,8 +26,8 @@ import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; import org.apache.iotdb.db.metadata.utils.ResourceByPathUtils; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java index adf995e9487b..8e54667e0add 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java @@ -25,7 +25,7 @@ import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode; import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java index 9bcf4f5807d7..30951af8622d 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTable.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.engine.memtable; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import java.util.HashMap; import java.util.Map; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java index 9d1bac5d9953..a6019faa3daf 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java @@ -44,8 +44,8 @@ import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.exception.WriteProcessRejectException; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; import org.apache.iotdb.db.metadata.path.AlignedPath; import org.apache.iotdb.db.metadata.utils.ResourceByPathUtils; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowNode; diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/AppendOnlyDiskSchemaManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/AppendOnlyDiskSchemaManager.java index 785644479144..0bbfdfab9b5b 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/AppendOnlyDiskSchemaManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/AppendOnlyDiskSchemaManager.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.metadata.idtable.deviceID.IStatefulDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; @@ -38,6 +39,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.RandomAccessFile; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -148,7 +150,6 @@ public void recover(IDTable idTable) { try (FileInputStream inputStream = new FileInputStream(dataFile)) { // read file version ReadWriteIOUtils.readString(inputStream); - while (inputStream.available() > 0) { DiskSchemaEntry cur = DiskSchemaEntry.deserialize(inputStream); if (!cur.deviceID.equals(DiskSchemaEntry.TOMBSTONE)) { @@ -158,11 +159,30 @@ public void recover(IDTable idTable) { TSEncoding.deserialize(cur.encoding), CompressionType.deserialize(cur.compressor), loc); + // if the system uses IStatefulDeviceID, need to recover the state of device id + if (IStatefulDeviceID.class.isAssignableFrom( + DeviceIDFactory.getInstance().getDeviceIDClass())) { + IStatefulDeviceID statefulIDeviceID = + (IStatefulDeviceID) + DeviceIDFactory.getInstance() + .getDeviceIDClass() + .getDeclaredConstructor() + .newInstance(); + statefulIDeviceID.recover( + cur.seriesKey.substring( + 0, cur.seriesKey.length() - cur.measurementName.length() - 1), + cur.deviceID); + } idTable.putSchemaEntry(cur.deviceID, cur.measurementName, schemaEntry, cur.isAligned); } loc += cur.entrySize; } - } catch (IOException | MetadataException e) { + } catch (IOException + | MetadataException + | IllegalAccessException + | InstantiationException + | NoSuchMethodException + | InvocationTargetException e) { logger.info("Last entry is incomplete, we will recover as much as we can."); try { outputStream.getChannel().truncate(loc); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTable.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTable.java index fa44865e4e75..df86da7d8143 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTable.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTable.java @@ -24,10 +24,10 @@ import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; import org.apache.iotdb.db.metadata.idtable.entry.TimeseriesID; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; @@ -133,6 +133,7 @@ void updateLastCache( throws MetadataException; /** clear id table and close file */ + @TestOnly void clear() throws IOException; /** @@ -143,6 +144,14 @@ void updateLastCache( */ DeviceEntry getDeviceEntry(String deviceName); + /** + * get device entry from device id + * + * @param deviceID device id of the device path + * @return device entry + */ + DeviceEntry getDeviceEntry(IDeviceID deviceID); + /** * get schema from device and measurements * @@ -162,13 +171,13 @@ void updateLastCache( /** * put schema entry to id table, currently used in recover * - * @param devicePath device path (can be device id formed path) + * @param deviceID device id * @param measurement measurement name * @param schemaEntry schema entry to put * @param isAligned is the device aligned */ void putSchemaEntry( - String devicePath, String measurement, SchemaEntry schemaEntry, boolean isAligned) + String deviceID, String measurement, SchemaEntry schemaEntry, boolean isAligned) throws MetadataException; /** diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableHashmapImpl.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableHashmapImpl.java index af3cb4d9bc03..b75f04b591fc 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableHashmapImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableHashmapImpl.java @@ -25,10 +25,11 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.StandAloneAutoIncDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.InsertMeasurementMNode; import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; import org.apache.iotdb.db.metadata.idtable.entry.TimeseriesID; @@ -72,6 +73,7 @@ public class IDTableHashmapImpl implements IDTable { /** disk schema manager to manage disk schema entry */ private IDiskSchemaManager IDiskSchemaManager; + /** iotdb config */ protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); @@ -307,10 +309,14 @@ public synchronized void updateLastCache( } @Override + @TestOnly public void clear() throws IOException { if (IDiskSchemaManager != null) { IDiskSchemaManager.close(); } + if (DeviceIDFactory.getInstance().getDeviceIDClass() == StandAloneAutoIncDeviceID.class) { + StandAloneAutoIncDeviceID.clear(); + } } /** @@ -321,7 +327,17 @@ public void clear() throws IOException { */ @Override public DeviceEntry getDeviceEntry(String deviceName) { - IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(deviceName); + return getDeviceEntry(DeviceIDFactory.getInstance().getDeviceID(deviceName)); + } + + /** + * get device entry from deviceID + * + * @param deviceID deviceID of the device path + * @return device entry + */ + @Override + public DeviceEntry getDeviceEntry(IDeviceID deviceID) { int slot = calculateSlot(deviceID); // reuse device entry in map @@ -355,6 +371,11 @@ public IMeasurementSchema getSeriesSchema(String deviceName, String measurementN schemaEntry.getCompressionType()); } + /** + * get all useful deviceEntry of idTable + * + * @return device entries + */ @Override public List getAllDeviceEntry() { List res = new ArrayList<>(); @@ -365,11 +386,20 @@ public List getAllDeviceEntry() { return res; } + /** + * put schema entry to id table, currently used in recover + * + * @param deviceID device id + * @param measurement measurement name + * @param schemaEntry schema entry to put + * @param isAligned is the device aligned + * @throws MetadataException + */ @Override public void putSchemaEntry( - String devicePath, String measurement, SchemaEntry schemaEntry, boolean isAligned) + String deviceID, String measurement, SchemaEntry schemaEntry, boolean isAligned) throws MetadataException { - DeviceEntry deviceEntry = getDeviceEntryWithAlignedCheck(devicePath, isAligned); + DeviceEntry deviceEntry = getDeviceEntryWithAlignedCheck(deviceID, isAligned); deviceEntry.putSchemaEntry(measurement, schemaEntry); } @@ -460,7 +490,7 @@ private IMeasurementMNode getOrCreateMeasurementIfNotExist( */ private DeviceEntry getDeviceEntryWithAlignedCheck(String deviceName, boolean isAligned) throws MetadataException { - IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(deviceName); + IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate(deviceName); int slot = calculateSlot(deviceID); DeviceEntry deviceEntry = idTables[slot].get(deviceID); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableManager.java index f60e18812aa1..235ca57bd1f2 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableManager.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/IDTableManager.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.metadata.PathNotExistException; import org.apache.iotdb.db.service.IoTDB; @@ -127,6 +128,7 @@ public synchronized IMeasurementSchema getSeriesSchema(String deviceName, String } /** clear id table map */ + @TestOnly public void clear() throws IOException { for (IDTable idTable : idTableMap.values()) { idTable.clear(); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/IDeviceID.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/IDeviceID.java similarity index 88% rename from server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/IDeviceID.java rename to server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/IDeviceID.java index f83dc525a528..881d41e9d2c6 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/IDeviceID.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/IDeviceID.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.metadata.idtable.entry; +package org.apache.iotdb.db.metadata.idtable.deviceID; import java.nio.ByteBuffer; @@ -29,7 +29,7 @@ public interface IDeviceID { * * @return string format device id */ - public String toStringID(); + String toStringID(); - public void serialize(ByteBuffer byteBuffer); + void serialize(ByteBuffer byteBuffer); } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/IStatefulDeviceID.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/IStatefulDeviceID.java new file mode 100644 index 000000000000..b89312b10114 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/IStatefulDeviceID.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.idtable.deviceID; + +/** stateful device id interface */ +public interface IStatefulDeviceID extends IDeviceID { + + /** + * recover state using devicePath and device id + * + * @param devicePath device path of the time series + * @param deviceID device id + */ + void recover(String devicePath, String deviceID); +} diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/PlainDeviceID.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/PlainDeviceID.java similarity index 97% rename from server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/PlainDeviceID.java rename to server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/PlainDeviceID.java index eac298cd7087..a799d55fd591 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/PlainDeviceID.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/PlainDeviceID.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.metadata.idtable.entry; +package org.apache.iotdb.db.metadata.idtable.deviceID; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SHA256DeviceID.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/SHA256DeviceID.java similarity index 97% rename from server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SHA256DeviceID.java rename to server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/SHA256DeviceID.java index 1042aa698573..fd88a4613b0a 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SHA256DeviceID.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/SHA256DeviceID.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.db.metadata.idtable.entry; +package org.apache.iotdb.db.metadata.idtable.deviceID; import org.apache.iotdb.db.metadata.idtable.IDTable; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; @@ -90,7 +90,7 @@ private void fromSHA256String(String deviceID) { * * @param deviceID device path */ - private void buildSHA256(String deviceID) { + protected void buildSHA256(String deviceID) { byte[] hashVal; int slot = calculateSlot(deviceID); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/StandAloneAutoIncDeviceID.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/StandAloneAutoIncDeviceID.java new file mode 100644 index 000000000000..0bcbb29f872c --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/deviceID/StandAloneAutoIncDeviceID.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.idtable.deviceID; + +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.localconfignode.LocalConfigNode; +import org.apache.iotdb.db.metadata.idtable.IDTable; +import org.apache.iotdb.db.metadata.idtable.IDTableManager; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Using auto-incrementing id as device id,A complete auto-increment id consists of schemaRegionID + * and autoIncrementID, where the upper 32 bits are schemaRegionID and the lower 32 bits are + * autoIncrementID + */ +public class StandAloneAutoIncDeviceID extends SHA256DeviceID implements IStatefulDeviceID { + + /** logger */ + private static Logger logger = LoggerFactory.getLogger(IDTable.class); + + // stand-alone auto-increment id uses LocalConfigNode to obtain schemaRegionId + private static LocalConfigNode configManager; + + // using map to maintain the mapping from schemaRegionId to list, each list + // maintains the auto-increment id of the schemaRegion + private static Map> deviceIDsMap; + + // if the device represented by devicePath is not written to the metadata module, use this + // constant instead of devicePath to generate a sha266 value of StandAloneAutoIncDeviceID instance + private static final String INVALID_DEVICE_PATH = "invalid.device.path"; + + // if the schemaRegionId==-1 of a StandAloneAutoIncDeviceID instance, it means that the device + // corresponding to the StandAloneAutoIncDeviceID instance does not exist + private static StandAloneAutoIncDeviceID deviceIdOfNonExistentDevice; + + // starting with 0,the maximum value is Integer.MAX_VALUE + int schemaRegionId; + + // starting with 0,the maximum value is Integer.MAX_VALUE + int autoIncrementID; + + static { + deviceIDsMap = new ConcurrentHashMap<>(); + configManager = LocalConfigNode.getInstance(); + setDeviceIdOfNonExistentDevice(); + } + + /** + * for the query path of a non-existing device, a deviceID with schemaRegion = -1 and + * autoIncrementID = 0 will be generated, and then stored in the deviceIDsMap. although it seems + * that all non-existing devicePaths will be converted into StandAloneAutoIncDeviceID objects with + * the same member variable value(schemaRegion,autoIncrementID), but due to the equality of + * StandAloneAutoIncDeviceID objects is determined by the sha256 hash value, so there is no + * adverse effect + */ + private static void setDeviceIdOfNonExistentDevice() { + deviceIdOfNonExistentDevice = new StandAloneAutoIncDeviceID(INVALID_DEVICE_PATH); + deviceIdOfNonExistentDevice.schemaRegionId = -1; + deviceIdOfNonExistentDevice.autoIncrementID = 0; + } + + public StandAloneAutoIncDeviceID() {} + + public StandAloneAutoIncDeviceID(String devicePath) { + super(devicePath); + } + + /** + * get a StandAloneAutoIncDeviceID instance, create it if it doesn't exist + * + * @param deviceID device path for insert/query, and device id for query + * @return a StandAloneAutoIncDeviceID instance + */ + public static StandAloneAutoIncDeviceID getDeviceIDWithAutoCreate(String deviceID) { + if (deviceID.startsWith("`") && deviceID.endsWith("`")) { + return fromAutoIncDeviceID(deviceID); + } else { + return buildDeviceID(deviceID); + } + } + + /** + * get a StandAloneAutoIncDeviceID instance, only for query + * + * @param deviceID device path or device id for query + * @return if the device exists, return a StandAloneAutoIncDeviceID instance, if it does not + * exist,return a StandAloneAutoIncDeviceID instance,the object is guaranteed to be different + * from the deviceID object of any device managed by the system (equals==false). + */ + public static StandAloneAutoIncDeviceID getDeviceID(String deviceID) { + if (deviceID.startsWith("`") && deviceID.endsWith("`")) { + return fromAutoIncDeviceID(deviceID); + } else { + return fromDevicePath(deviceID); + } + } + + /** + * get device id from a standAloneAutoIncDeviceID + * + * @param deviceID StandAloneAutoIncDeviceID deviceID, like: "`1`" + * @return a standAloneAutoIncDeviceID instance + */ + private static StandAloneAutoIncDeviceID fromAutoIncDeviceID(String deviceID) { + deviceID = deviceID.substring(1, deviceID.length() - 1); + long id = Long.parseLong(deviceID); + int schemaRegionId = (int) (id >>> 32); + int autoIncrementID = (int) id; + if (schemaRegionId == -1) { + return deviceIdOfNonExistentDevice; + } + List deviceIDs = deviceIDsMap.get(schemaRegionId); + synchronized (deviceIDs) { + return (StandAloneAutoIncDeviceID) deviceIDs.get(autoIncrementID); + } + } + + /** + * get device id from a device path + * + * @param devicePath device path, like: "root.sg.x.d1" + * @return a standAloneAutoIncDeviceID instance + */ + private static StandAloneAutoIncDeviceID fromDevicePath(String devicePath) { + try { + // use idTable to determine whether the device has been created + IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath(devicePath)); + StandAloneAutoIncDeviceID deviceID = new StandAloneAutoIncDeviceID(devicePath); + if (idTable.getDeviceEntry(deviceID) != null) { + deviceID = (StandAloneAutoIncDeviceID) idTable.getDeviceEntry(deviceID).getDeviceID(); + return deviceID; + } else { + return deviceIdOfNonExistentDevice; + } + } catch (IllegalPathException e) { + logger.info(e.getMessage()); + return deviceIdOfNonExistentDevice; + } + } + + /** + * get device id from a device path, if the device represented by the path does not exist, a + * StandAloneAutoIncDeviceID instance is generated for the path + * + * @param devicePath device path, like: "root.sg.x.d1" + * @return a standAloneAutoIncDeviceID instance + */ + private static StandAloneAutoIncDeviceID buildDeviceID(String devicePath) { + try { + PartialPath path = new PartialPath(devicePath); + // use idTable to determine whether the device has been created + IDTable idTable = IDTableManager.getInstance().getIDTable(path); + StandAloneAutoIncDeviceID deviceID = new StandAloneAutoIncDeviceID(devicePath); + // this device is added for the first time + if (idTable.getDeviceEntry(deviceID) == null) { + SchemaRegionId schemaRegionId = configManager.getBelongedSchemaRegionId(path); + deviceID.schemaRegionId = schemaRegionId.getId(); + List deviceIDs = + deviceIDsMap.computeIfAbsent(deviceID.schemaRegionId, integer -> new ArrayList<>()); + synchronized (deviceIDs) { + deviceID.autoIncrementID = deviceIDs.size(); + deviceIDs.add(deviceID.autoIncrementID, deviceID); + } + } else { + deviceID = (StandAloneAutoIncDeviceID) idTable.getDeviceEntry(deviceID).getDeviceID(); + } + return deviceID; + } catch (MetadataException e) { + logger.error(e.getMessage()); + return null; + } + } + + @Override + public int hashCode() { + return super.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof StandAloneAutoIncDeviceID)) { + return false; + } + return super.equals(o); + } + + @Override + public String toString() { + return "AutoIncrementDeviceID{" + + "l1=" + + l1 + + ", l2=" + + l2 + + ", l3=" + + l3 + + ", l4=" + + l4 + + ", schemaRegionId=" + + schemaRegionId + + ", autoIncrementID=" + + autoIncrementID + + '}'; + } + + @Override + public String toStringID() { + long stringID = (long) schemaRegionId << 32; + stringID |= autoIncrementID; + return "`" + stringID + '`'; + } + + @Override + public void serialize(ByteBuffer byteBuffer) { + super.serialize(byteBuffer); + ReadWriteIOUtils.write(schemaRegionId, byteBuffer); + ReadWriteIOUtils.write(autoIncrementID, byteBuffer); + } + + public static StandAloneAutoIncDeviceID deserialize(ByteBuffer byteBuffer) { + StandAloneAutoIncDeviceID autoIncrementDeviceID = new StandAloneAutoIncDeviceID(); + autoIncrementDeviceID.l1 = ReadWriteIOUtils.readLong(byteBuffer); + autoIncrementDeviceID.l2 = ReadWriteIOUtils.readLong(byteBuffer); + autoIncrementDeviceID.l3 = ReadWriteIOUtils.readLong(byteBuffer); + autoIncrementDeviceID.l4 = ReadWriteIOUtils.readLong(byteBuffer); + autoIncrementDeviceID.schemaRegionId = ReadWriteIOUtils.readInt(byteBuffer); + autoIncrementDeviceID.autoIncrementID = ReadWriteIOUtils.readInt(byteBuffer); + return autoIncrementDeviceID; + } + + /** + * recover deviceIDsMap + * + * @param devicePath device path of the time series, like: "root.sg.x.d1" + * @param deviceID device id, like: "`1`" + */ + @Override + public void recover(String devicePath, String deviceID) { + buildSHA256(devicePath); + deviceID = deviceID.substring(1, deviceID.length() - 1); + long id = Long.parseLong(deviceID); + this.schemaRegionId = (int) (id >>> 32); + this.autoIncrementID = (int) id; + List deviceIDs = + deviceIDsMap.computeIfAbsent(schemaRegionId, integer -> new ArrayList<>()); + // if there is out-of-order data, write the deviceID to the correct index of the array + synchronized (deviceIDs) { + if (autoIncrementID < deviceIDs.size() && deviceIDs.get(autoIncrementID) != null) return; + for (int i = deviceIDs.size(); i < autoIncrementID; i++) { + deviceIDs.add(i, null); + } + deviceIDs.add(autoIncrementID, this); + } + } + + @TestOnly + public static void reset() { + deviceIDsMap.clear(); + configManager = LocalConfigNode.getInstance(); + setDeviceIdOfNonExistentDevice(); + } + + @TestOnly + public static void clear() { + deviceIDsMap.clear(); + configManager = null; + deviceIdOfNonExistentDevice = null; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceEntry.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceEntry.java index 3ef77edec71b..0f42b41ab463 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceEntry.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceEntry.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.metadata.idtable.entry; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; + import java.util.HashMap; import java.util.Map; import java.util.Objects; diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDFactory.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDFactory.java index 332183cb3bec..4033006f6f3a 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDFactory.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDFactory.java @@ -22,6 +22,10 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.PlainDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.SHA256DeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.StandAloneAutoIncDeviceID; import java.util.function.Function; @@ -29,6 +33,8 @@ public class DeviceIDFactory { Function getDeviceIDFunction; + Function getDeviceIDWithAutoCreateFunction; + // region DeviceIDFactory Singleton private static class DeviceIDFactoryHolder { @@ -49,15 +55,25 @@ public static DeviceIDFactory getInstance() { } private DeviceIDFactory() { - if (IoTDBDescriptor.getInstance().getConfig().isEnableIDTable() - && IoTDBDescriptor.getInstance() - .getConfig() - .getDeviceIDTransformationMethod() - .equals("SHA256")) { - getDeviceIDFunction = SHA256DeviceID::new; - } else { - getDeviceIDFunction = PlainDeviceID::new; + if (IoTDBDescriptor.getInstance().getConfig().isEnableIDTable()) { + if (IoTDBDescriptor.getInstance() + .getConfig() + .getDeviceIDTransformationMethod() + .equals("SHA256")) { + getDeviceIDFunction = SHA256DeviceID::new; + getDeviceIDWithAutoCreateFunction = SHA256DeviceID::new; + return; + } else if (IoTDBDescriptor.getInstance() + .getConfig() + .getDeviceIDTransformationMethod() + .equals("AutoIncrement_INT")) { + getDeviceIDFunction = StandAloneAutoIncDeviceID::getDeviceID; + getDeviceIDWithAutoCreateFunction = StandAloneAutoIncDeviceID::getDeviceIDWithAutoCreate; + return; + } } + getDeviceIDFunction = PlainDeviceID::new; + getDeviceIDWithAutoCreateFunction = PlainDeviceID::new; } // endregion @@ -81,17 +97,65 @@ public IDeviceID getDeviceID(String devicePath) { return getDeviceIDFunction.apply(devicePath); } + /** + * get and set device id by full path + * + * @param devicePath device path of the timeseries + * @return device id of the timeseries + */ + public IDeviceID getDeviceIDWithAutoCreate(PartialPath devicePath) { + return getDeviceIDWithAutoCreateFunction.apply(devicePath.toString()); + } + + /** + * get and set device id by full path + * + * @param devicePath device path of the timeseries + * @return device id of the timeseries + */ + public IDeviceID getDeviceIDWithAutoCreate(String devicePath) { + return getDeviceIDWithAutoCreateFunction.apply(devicePath); + } + /** reset id method */ @TestOnly public void reset() { - if (IoTDBDescriptor.getInstance().getConfig().isEnableIDTable() - && IoTDBDescriptor.getInstance() - .getConfig() - .getDeviceIDTransformationMethod() - .equals("SHA256")) { - getDeviceIDFunction = SHA256DeviceID::new; - } else { - getDeviceIDFunction = PlainDeviceID::new; + if (IoTDBDescriptor.getInstance().getConfig().isEnableIDTable()) { + if (IoTDBDescriptor.getInstance() + .getConfig() + .getDeviceIDTransformationMethod() + .equals("SHA256")) { + getDeviceIDFunction = SHA256DeviceID::new; + getDeviceIDWithAutoCreateFunction = SHA256DeviceID::new; + return; + } else if (IoTDBDescriptor.getInstance() + .getConfig() + .getDeviceIDTransformationMethod() + .equals("AutoIncrement_INT")) { + getDeviceIDFunction = StandAloneAutoIncDeviceID::getDeviceID; + getDeviceIDWithAutoCreateFunction = StandAloneAutoIncDeviceID::getDeviceIDWithAutoCreate; + StandAloneAutoIncDeviceID.reset(); + return; + } + } + getDeviceIDFunction = PlainDeviceID::new; + getDeviceIDWithAutoCreateFunction = PlainDeviceID::new; + } + + public Class getDeviceIDClass() { + if (IoTDBDescriptor.getInstance().getConfig().isEnableIDTable()) { + if (IoTDBDescriptor.getInstance() + .getConfig() + .getDeviceIDTransformationMethod() + .equals("SHA256")) { + return SHA256DeviceID.class; + } else if (IoTDBDescriptor.getInstance() + .getConfig() + .getDeviceIDTransformationMethod() + .equals("AutoIncrement_INT")) { + return StandAloneAutoIncDeviceID.class; + } } + return PlainDeviceID.class; } } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SchemaEntry.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SchemaEntry.java index bf95324a4b28..589a7d198697 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SchemaEntry.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SchemaEntry.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.metadata.idtable.IDiskSchemaManager; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.metadata.lastCache.container.ILastCacheContainer; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/TimeseriesID.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/TimeseriesID.java index 143ddbc3b37e..2d8ce79e2615 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/TimeseriesID.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/TimeseriesID.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.metadata.idtable.entry; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; /** A po class contains device id and measurement, represents a timeseries */ public class TimeseriesID { diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/utils/ResourceByPathUtils.java b/server/src/main/java/org/apache/iotdb/db/metadata/utils/ResourceByPathUtils.java index 74b8c0b59dc7..86aac6721502 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/utils/ResourceByPathUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/utils/ResourceByPathUtils.java @@ -33,8 +33,8 @@ import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; import org.apache.iotdb.db.metadata.path.AlignedPath; import org.apache.iotdb.db.metadata.path.MeasurementPath; import org.apache.iotdb.db.query.context.QueryContext; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertNode.java index 610628b33434..9cfcfccccf1e 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertNode.java @@ -23,7 +23,7 @@ import org.apache.iotdb.consensus.multileader.wal.ConsensusReqReader; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.mpp.common.schematree.ISchemaTree; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.mpp.plan.planner.plan.node.WritePlanNode; diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java index a5ef81ff12a5..1333692df520 100644 --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; import org.apache.iotdb.db.qp.logical.Operator; import org.apache.iotdb.db.qp.physical.PhysicalPlan; diff --git a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/UnsealedTsFileRecoverPerformer.java index 926e54a75b22..631a6cfe7e23 100644 --- a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/UnsealedTsFileRecoverPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/UnsealedTsFileRecoverPerformer.java @@ -28,7 +28,7 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.DataRegionException; import org.apache.iotdb.db.metadata.idtable.IDTable; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.qp.physical.crud.DeletePlan; import org.apache.iotdb.db.qp.physical.crud.InsertPlan; diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableFlushTimeTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableFlushTimeTest.java index cb48267bbf72..bae5e4a089af 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableFlushTimeTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableFlushTimeTest.java @@ -70,19 +70,19 @@ public void before() { isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement_INT"); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile); - EnvironmentUtils.cleanEnv(); } @Test diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableLogFileTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableLogFileTest.java index eb2b337518fd..f67bd0499add 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableLogFileTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableLogFileTest.java @@ -57,19 +57,19 @@ public void before() { isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement_INT"); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile); - EnvironmentUtils.cleanEnv(); } @Test diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRecoverTest.java index bcacb1607358..cd0fda5da09a 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRecoverTest.java @@ -63,32 +63,32 @@ public void before() { isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement_INT"); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile); - - EnvironmentUtils.cleanEnv(); } @Test public void testRecover() throws Exception { - insertDataInMemoryWithTablet(false); - insertDataInMemoryWithRecord(false); - + insertDataInMemoryWithTablet("root.isp1.d2", false); + insertDataInMemoryWithRecord("root.isp1.d1", false); + insertDataInMemoryWithTablet("root.isp2.d2", false); + insertDataInMemoryWithRecord("root.isp2.d1", false); PlanExecutor executor = new PlanExecutor(); PhysicalPlan flushPlan = processor.parseSQLToPhysicalPlan("flush"); executor.processNonQuery(flushPlan); - IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp")); + IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp1")); List memoryList = idTable.getAllDeviceEntry(); // restart @@ -100,22 +100,40 @@ public void testRecover() throws Exception { // check id table fields - idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp.d1")); + idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp1.d1")); List recoverList = idTable.getAllDeviceEntry(); assertEquals(memoryList, recoverList); + + idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp2")); + memoryList = idTable.getAllDeviceEntry(); + + // restart + try { + EnvironmentUtils.restartDaemon(); + } catch (Exception e) { + Assert.fail(); + } + + // check id table fields + + idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp2.d1")); + recoverList = idTable.getAllDeviceEntry(); + + assertEquals(memoryList, recoverList); } @Test public void testRecoverAligned() throws Exception { - insertDataInMemoryWithTablet(true); - insertDataInMemoryWithRecord(false); - + insertDataInMemoryWithTablet("root.isp1.d2", true); + insertDataInMemoryWithRecord("root.isp1.d1", true); + insertDataInMemoryWithTablet("root.isp2.d2", true); + insertDataInMemoryWithRecord("root.isp2.d1", true); PlanExecutor executor = new PlanExecutor(); PhysicalPlan flushPlan = processor.parseSQLToPhysicalPlan("flush"); executor.processNonQuery(flushPlan); - IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp")); + IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp1")); List memoryList = idTable.getAllDeviceEntry(); // restart @@ -127,13 +145,30 @@ public void testRecoverAligned() throws Exception { // check id table fields - idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp.d1")); + idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp1.d1")); List recoverList = idTable.getAllDeviceEntry(); assertEquals(memoryList, recoverList); + + idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp2")); + memoryList = idTable.getAllDeviceEntry(); + + // restart + try { + EnvironmentUtils.restartDaemon(); + } catch (Exception e) { + Assert.fail(); + } + + // check id table fields + + idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp2.d1")); + recoverList = idTable.getAllDeviceEntry(); + + assertEquals(memoryList, recoverList); } - private void insertDataInMemoryWithRecord(boolean isAligned) + private void insertDataInMemoryWithRecord(String storageGroupPath, boolean isAligned) throws IllegalPathException, QueryProcessException { long time = 100L; TSDataType[] dataTypes = @@ -156,7 +191,7 @@ private void insertDataInMemoryWithRecord(boolean isAligned) InsertRowPlan insertRowPlan = new InsertRowPlan( - new PartialPath("root.isp.d1"), + new PartialPath(storageGroupPath), time, new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, dataTypes, @@ -167,7 +202,7 @@ private void insertDataInMemoryWithRecord(boolean isAligned) executor.insert(insertRowPlan); } - private void insertDataInMemoryWithTablet(boolean isAligned) + private void insertDataInMemoryWithTablet(String storageGroupPath, boolean isAligned) throws IllegalPathException, QueryProcessException { long[] times = new long[] {110L, 111L, 112L, 113L}; List dataTypes = new ArrayList<>(); @@ -197,7 +232,7 @@ private void insertDataInMemoryWithTablet(boolean isAligned) InsertTabletPlan tabletPlan = new InsertTabletPlan( - new PartialPath("root.isp.d2"), + new PartialPath(storageGroupPath), new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, dataTypes); tabletPlan.setTimes(times); diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableResourceControlTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableResourceControlTest.java index 35c9ff4f1b13..3d6b4c6278b5 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableResourceControlTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableResourceControlTest.java @@ -27,8 +27,8 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileProcessor; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; import org.apache.iotdb.db.qp.Planner; import org.apache.iotdb.db.qp.executor.PlanExecutor; import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; @@ -65,17 +65,17 @@ public void before() { IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement_INT"); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); - EnvironmentUtils.cleanEnv(); } @Test diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRestartTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRestartTest.java index f4801710df50..517be7312219 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRestartTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRestartTest.java @@ -66,26 +66,29 @@ public void before() { isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement_INT"); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile); - - EnvironmentUtils.cleanEnv(); } @Test public void testRawDataQueryAfterRestart() throws Exception { - insertDataInMemoryWithTablet(); - insertDataInMemoryWithRecord(); + String sg1 = "root.isp1"; + String sg2 = "root.isp2"; + insertDataInMemoryWithTablet(sg1 + ".d1"); + insertDataInMemoryWithRecord(sg1 + ".d1"); + insertDataInMemoryWithTablet(sg2 + ".d1"); + insertDataInMemoryWithRecord(sg2 + ".d1"); // restart try { @@ -95,7 +98,8 @@ public void testRawDataQueryAfterRestart() throws Exception { } PlanExecutor executor = new PlanExecutor(); - QueryPlan queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from root.isp.d1"); + QueryPlan queryPlan = + (QueryPlan) processor.parseSQLToPhysicalPlan("select * from " + sg1 + ".d1"); QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); Assert.assertEquals(6, dataSet.getPaths().size()); int count = 0; @@ -107,6 +111,18 @@ public void testRawDataQueryAfterRestart() throws Exception { assertEquals(5, count); + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from " + sg2 + ".d1"); + dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + Assert.assertEquals(6, dataSet.getPaths().size()); + count = 0; + while (dataSet.hasNext()) { + RowRecord record = dataSet.next(); + System.out.println(record); + count++; + } + + assertEquals(5, count); + // flush and test again PhysicalPlan flushPlan = processor.parseSQLToPhysicalPlan("flush"); executor.processNonQuery(flushPlan); @@ -118,7 +134,7 @@ public void testRawDataQueryAfterRestart() throws Exception { } executor = new PlanExecutor(); - queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from root.isp.d1"); + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from " + sg1 + ".d1"); dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); Assert.assertEquals(6, dataSet.getPaths().size()); count = 0; @@ -129,9 +145,25 @@ public void testRawDataQueryAfterRestart() throws Exception { } assertEquals(5, count); + + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from " + sg2 + ".d1"); + dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + Assert.assertEquals(6, dataSet.getPaths().size()); + count = 0; + while (dataSet.hasNext()) { + RowRecord record = dataSet.next(); + System.out.println(record); + count++; + } + + assertEquals(5, count); + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from " + sg2 + ".d2"); + dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + Assert.assertEquals(0, dataSet.getPaths().size()); } - private void insertDataInMemoryWithRecord() throws IllegalPathException, QueryProcessException { + private void insertDataInMemoryWithRecord(String storageGroupPath) + throws IllegalPathException, QueryProcessException { long time = 100L; TSDataType[] dataTypes = new TSDataType[] { @@ -153,7 +185,7 @@ private void insertDataInMemoryWithRecord() throws IllegalPathException, QueryPr InsertRowPlan insertRowPlan = new InsertRowPlan( - new PartialPath("root.isp.d1"), + new PartialPath(storageGroupPath), time, new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, dataTypes, @@ -163,7 +195,8 @@ private void insertDataInMemoryWithRecord() throws IllegalPathException, QueryPr executor.insert(insertRowPlan); } - private void insertDataInMemoryWithTablet() throws IllegalPathException, QueryProcessException { + private void insertDataInMemoryWithTablet(String storageGroupPath) + throws IllegalPathException, QueryProcessException { long[] times = new long[] {110L, 111L, 112L, 113L}; List dataTypes = new ArrayList<>(); dataTypes.add(TSDataType.DOUBLE.ordinal()); @@ -192,7 +225,7 @@ private void insertDataInMemoryWithTablet() throws IllegalPathException, QueryPr InsertTabletPlan tabletPlan = new InsertTabletPlan( - new PartialPath("root.isp.d1"), + new PartialPath(storageGroupPath), new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, dataTypes); tabletPlan.setTimes(times); diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableTest.java index 2ea9f8cd35a8..a66b2cf66efa 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableTest.java @@ -27,9 +27,9 @@ import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.metadata.LocalSchemaProcessor; +import org.apache.iotdb.db.metadata.idtable.deviceID.IDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; -import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; import org.apache.iotdb.db.metadata.lastCache.container.ILastCacheContainer; import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; @@ -88,19 +88,19 @@ public void before() { isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement_INT"); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile); - EnvironmentUtils.cleanEnv(); } @Test @@ -614,7 +614,7 @@ public void testGetDiskSchemaEntries() { String sgPath = "root.laptop"; for (int i = 0; i < 10; i++) { String devicePath = sgPath + ".d" + i; - IDeviceID iDeviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath); + IDeviceID iDeviceID = DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate(devicePath); String measurement = "s" + i; idTable.putSchemaEntry( devicePath, diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/InsertWithIDTableTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/InsertWithIDTableTest.java index 15f677a6a1bb..66ff2836eff3 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/InsertWithIDTableTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/InsertWithIDTableTest.java @@ -75,17 +75,17 @@ public void before() { IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement_INT"); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); - EnvironmentUtils.cleanEnv(); } @Test @@ -755,5 +755,10 @@ public void testInsertMultiTabletPlan() RowRecord record = dataSet.next(); assertEquals(60, record.getFields().size()); } + + // query for records that do not exist + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select * from root.multi.d11"); + dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + assertEquals(0, dataSet.getPaths().size()); } } diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/LastQueryWithIDTable.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/LastQueryWithIDTable.java index 33bbd75d0a09..dbc19c1977be 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/LastQueryWithIDTable.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/LastQueryWithIDTable.java @@ -65,12 +65,18 @@ public class LastQueryWithIDTable { Set retSet = new HashSet<>( Arrays.asList( - "113\troot.isp.d1.s3\t100003\tINT64", - "113\troot.isp.d1.s4\t1003\tINT32", - "113\troot.isp.d1.s5\tfalse\tBOOLEAN", - "113\troot.isp.d1.s6\tmm3\tTEXT", - "113\troot.isp.d1.s1\t13.0\tDOUBLE", - "113\troot.isp.d1.s2\t23.0\tFLOAT")); + "113\troot.isp1.d1.s3\t100003\tINT64", + "113\troot.isp1.d1.s4\t1003\tINT32", + "113\troot.isp1.d1.s5\tfalse\tBOOLEAN", + "113\troot.isp1.d1.s6\tmm3\tTEXT", + "113\troot.isp1.d1.s1\t13.0\tDOUBLE", + "113\troot.isp1.d1.s2\t23.0\tFLOAT", + "113\troot.isp2.d1.s3\t100003\tINT64", + "113\troot.isp2.d1.s4\t1003\tINT32", + "113\troot.isp2.d1.s5\tfalse\tBOOLEAN", + "113\troot.isp2.d1.s6\tmm3\tTEXT", + "113\troot.isp2.d1.s1\t13.0\tDOUBLE", + "113\troot.isp2.d1.s2\t23.0\tFLOAT")); @Before public void before() { @@ -82,18 +88,18 @@ public void before() { IoTDBDescriptor.getInstance().getConfig().setEnableLastCache(false); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement_INT"); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); IoTDBDescriptor.getInstance().getConfig().setEnableLastCache(originalEnableCache); - EnvironmentUtils.cleanEnv(); } @Test @@ -101,11 +107,12 @@ public void testLastCacheQueryWithoutCache() throws QueryProcessException, MetadataException, InterruptedException, QueryFilterOptimizationException, StorageEngineException, IOException { - insertDataInMemory(); + insertDataInMemory("root.isp1.d1"); + insertDataInMemory("root.isp2.d1"); PlanExecutor executor = new PlanExecutor(); QueryPlan queryPlan = - (QueryPlan) processor.parseSQLToPhysicalPlan("select last * from root.isp.d1"); + (QueryPlan) processor.parseSQLToPhysicalPlan("select last * from root.isp1.d1"); QueryDataSet dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); Assert.assertEquals(3, dataSet.getPaths().size()); int count = 0; @@ -115,12 +122,26 @@ public void testLastCacheQueryWithoutCache() count++; } - assertEquals(retSet.size(), count); + assertEquals(6, count); + + executor = new PlanExecutor(); + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select last * from root.isp2.d1"); + dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + Assert.assertEquals(3, dataSet.getPaths().size()); + count = 0; + while (dataSet.hasNext()) { + RowRecord record = dataSet.next(); + assertTrue(retSet.contains(record.toString())); + count++; + } + + assertEquals(6, count); // flush and test again PhysicalPlan flushPlan = processor.parseSQLToPhysicalPlan("flush"); executor.processNonQuery(flushPlan); + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select last * from root.isp1.d1"); dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); Assert.assertEquals(3, dataSet.getPaths().size()); count = 0; @@ -129,16 +150,37 @@ public void testLastCacheQueryWithoutCache() assertTrue(retSet.contains(record.toString())); count++; } - assertEquals(retSet.size(), count); + assertEquals(6, count); // assert id table is not refresh assertNull( IDTableManager.getInstance() - .getIDTable(new PartialPath("root.isp.d1")) - .getLastCache(new TimeseriesID(new PartialPath("root.isp.d1.s1")))); + .getIDTable(new PartialPath("root.isp1.d1")) + .getLastCache(new TimeseriesID(new PartialPath("root.isp1.d1.s1")))); + + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select last * from root.isp2.d1"); + dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + Assert.assertEquals(3, dataSet.getPaths().size()); + count = 0; + while (dataSet.hasNext()) { + RowRecord record = dataSet.next(); + assertTrue(retSet.contains(record.toString())); + count++; + } + assertEquals(6, count); + + // assert id table is not refresh + assertNull( + IDTableManager.getInstance() + .getIDTable(new PartialPath("root.isp2.d1")) + .getLastCache(new TimeseriesID(new PartialPath("root.isp2.d1.s1")))); + + queryPlan = (QueryPlan) processor.parseSQLToPhysicalPlan("select last * from root.isp2.d2"); + dataSet = executor.processQuery(queryPlan, EnvironmentUtils.TEST_QUERY_CONTEXT); + Assert.assertEquals(0, dataSet.getPaths().size()); } - private void insertDataInMemory() throws IllegalPathException, QueryProcessException { + private void insertDataInMemory(String path) throws IllegalPathException, QueryProcessException { long[] times = new long[] {110L, 111L, 112L, 113L}; List dataTypes = new ArrayList<>(); dataTypes.add(TSDataType.DOUBLE.ordinal()); @@ -167,9 +209,7 @@ private void insertDataInMemory() throws IllegalPathException, QueryProcessExcep InsertTabletPlan tabletPlan = new InsertTabletPlan( - new PartialPath("root.isp.d1"), - new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, - dataTypes); + new PartialPath(path), new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, dataTypes); tabletPlan.setTimes(times); tabletPlan.setColumns(columns); tabletPlan.setRowCount(times.length); diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/QueryAlignedTimeseriesWithIDTableTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/QueryAlignedTimeseriesWithIDTableTest.java index 22c77325b18f..356eee31098f 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/QueryAlignedTimeseriesWithIDTableTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/QueryAlignedTimeseriesWithIDTableTest.java @@ -142,17 +142,17 @@ public void before() { IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement_INT"); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); - EnvironmentUtils.cleanEnv(); } @Test diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/QueryWithIDTableTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/QueryWithIDTableTest.java index ca44e1fc8f7f..7da39d469e1f 100644 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/QueryWithIDTableTest.java +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/QueryWithIDTableTest.java @@ -145,19 +145,19 @@ public void before() { isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); - IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement_INT"); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true); EnvironmentUtils.envSetUp(); } @After public void clean() throws IOException, StorageEngineException { + EnvironmentUtils.cleanEnv(); IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); IoTDBDescriptor.getInstance() .getConfig() .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile); - EnvironmentUtils.cleanEnv(); } @Test diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/deviceID/DeviceIDTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/deviceID/DeviceIDTest.java new file mode 100644 index 000000000000..6d7ac6794722 --- /dev/null +++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/deviceID/DeviceIDTest.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.metadata.idtable.deviceID; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.utils.EnvironmentUtils; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +public class DeviceIDTest { + + private boolean isEnableIDTable = false; + + private String originalDeviceIDTransformationMethod = null; + + @Before + public void setUp() throws Exception { + isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable(); + originalDeviceIDTransformationMethod = + IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod(); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("AutoIncrement_INT"); + EnvironmentUtils.envSetUp(); + } + + @After + public void tearDown() throws Exception { + EnvironmentUtils.cleanEnv(); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); + IoTDBDescriptor.getInstance() + .getConfig() + .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); + } + + @Test + public void deviceIDBuildTest() throws IllegalPathException { + PartialPath partialPath1 = new PartialPath("root.sg1.d1.s1"); + PartialPath partialPath2 = new PartialPath("root.sg1.d1.s2"); + PartialPath partialPath3 = new PartialPath("root.sg1.d2.s1"); + + IDeviceID deviceID1 = + DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate(partialPath1.getDevicePath()); + IDeviceID deviceID2 = + DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate(partialPath2.getDevicePath()); + IDeviceID deviceID3 = + DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate(partialPath3.getDevicePath()); + + assertEquals(deviceID1, deviceID2); + assertNotEquals(deviceID1, deviceID3); + assertNotEquals(deviceID2, deviceID3); + } + + @Test + public void testHashCode() { + IDeviceID deviceID1 = DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate("root.sg1.x.d1"); + IDeviceID deviceID2 = DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate("root.sg1.x.d1"); + IDeviceID deviceID3 = DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate("root.sg1.x.d2"); + assertEquals(deviceID1.hashCode(), deviceID2.hashCode()); + assertNotEquals(deviceID1.hashCode(), deviceID3.hashCode()); + IDeviceID deviceID4 = DeviceIDFactory.getInstance().getDeviceID(deviceID1.toStringID()); + IDeviceID deviceID5 = DeviceIDFactory.getInstance().getDeviceID(deviceID2.toStringID()); + IDeviceID deviceID6 = DeviceIDFactory.getInstance().getDeviceID(deviceID3.toStringID()); + assertEquals(deviceID1.hashCode(), deviceID4.hashCode()); + assertEquals(deviceID1.hashCode(), deviceID5.hashCode()); + assertEquals(deviceID3.hashCode(), deviceID6.hashCode()); + + deviceID1 = DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate("root.sg2.x.d1"); + deviceID2 = DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate("root.sg2.x.d1"); + deviceID3 = DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate("root.sg2.x.d2"); + assertEquals(deviceID1.hashCode(), deviceID2.hashCode()); + assertNotEquals(deviceID1.hashCode(), deviceID3.hashCode()); + deviceID4 = DeviceIDFactory.getInstance().getDeviceID(deviceID1.toStringID()); + deviceID5 = DeviceIDFactory.getInstance().getDeviceID(deviceID2.toStringID()); + deviceID6 = DeviceIDFactory.getInstance().getDeviceID(deviceID3.toStringID()); + assertEquals(deviceID1.hashCode(), deviceID4.hashCode()); + assertEquals(deviceID1.hashCode(), deviceID5.hashCode()); + assertEquals(deviceID3.hashCode(), deviceID6.hashCode()); + } + + @Test + public void testEquals() throws MetadataException { + IDeviceID deviceID1 = DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate("root.sg1.x.d1"); + IDeviceID deviceID2 = DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate("root.sg1.x.d1"); + IDeviceID deviceID3 = DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate("root.sg1.x.d2"); + assertEquals(deviceID1, deviceID2); + assertNotEquals(deviceID1, deviceID3); + IDeviceID deviceID4 = DeviceIDFactory.getInstance().getDeviceID(deviceID1.toStringID()); + IDeviceID deviceID5 = DeviceIDFactory.getInstance().getDeviceID(deviceID2.toStringID()); + IDeviceID deviceID6 = DeviceIDFactory.getInstance().getDeviceID(deviceID3.toStringID()); + assertEquals(deviceID1, deviceID4); + assertEquals(deviceID1, deviceID5); + assertEquals(deviceID3, deviceID6); + + deviceID1 = DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate("root.sg2.x.d1"); + deviceID2 = DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate("root.sg2.x.d1"); + deviceID3 = DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate("root.sg2.x.d2"); + assertEquals(deviceID1, deviceID2); + assertNotEquals(deviceID1, deviceID3); + deviceID4 = DeviceIDFactory.getInstance().getDeviceID(deviceID1.toStringID()); + deviceID5 = DeviceIDFactory.getInstance().getDeviceID(deviceID2.toStringID()); + deviceID6 = DeviceIDFactory.getInstance().getDeviceID(deviceID3.toStringID()); + assertEquals(deviceID1, deviceID4); + assertEquals(deviceID1, deviceID5); + assertEquals(deviceID3, deviceID6); + } + + @Test + public void testSerializeAndDeserialize() throws MetadataException { + for (int i = 1; i < 10; i++) { + ByteBuffer byteBuffer = ByteBuffer.allocate(100); + IDeviceID deviceID = + DeviceIDFactory.getInstance().getDeviceIDWithAutoCreate("root.sg.x.d" + i); + deviceID.serialize(byteBuffer); + byteBuffer.flip(); + IDeviceID deviceID1 = StandAloneAutoIncDeviceID.deserialize(byteBuffer); + assertEquals(deviceID, deviceID1); + } + } +} diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDTest.java deleted file mode 100644 index 54ebc0afeba6..000000000000 --- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.metadata.idtable.entry; - -import org.apache.iotdb.commons.exception.IllegalPathException; -import org.apache.iotdb.commons.path.PartialPath; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; - -public class DeviceIDTest { - @Test - public void deviceIDBuildTest() throws IllegalPathException { - PartialPath partialPath1 = new PartialPath("root.sg1.d1.s1"); - PartialPath partialPath2 = new PartialPath("root.sg1.d1.s2"); - PartialPath partialPath3 = new PartialPath("root.sg1.d2.s1"); - - IDeviceID deviceID1 = DeviceIDFactory.getInstance().getDeviceID(partialPath1.getDevicePath()); - IDeviceID deviceID2 = DeviceIDFactory.getInstance().getDeviceID(partialPath2.getDevicePath()); - IDeviceID deviceID3 = DeviceIDFactory.getInstance().getDeviceID(partialPath3.getDevicePath()); - - assertEquals(deviceID1, deviceID2); - assertNotEquals(deviceID1, deviceID3); - assertNotEquals(deviceID2, deviceID3); - } -}