diff --git a/example/influxdb-protocol-example/src/main/java/org/apache/iotdb/influxdb/InfluxDBExample.java b/example/influxdb-protocol-example/src/main/java/org/apache/iotdb/influxdb/InfluxDBExample.java index 64a5d178226a..cdb884dfc1b2 100644 --- a/example/influxdb-protocol-example/src/main/java/org/apache/iotdb/influxdb/InfluxDBExample.java +++ b/example/influxdb-protocol-example/src/main/java/org/apache/iotdb/influxdb/InfluxDBExample.java @@ -102,7 +102,7 @@ private static void queryData() { query = new Query( - "select count(temperature),first(temperature),last(temperature),max(temperature),mean(temperature),median(temperature),min(temperature),mode(temperature),spread(temperature),stddev(temperature),sum(temperature) from student where ((workshop=\"A1\" and production=\"B1\" and cell =\"C1\" ) or temperature< 15 )", + "select count(temperature),first(temperature),last(temperature),max(temperature),mean(temperature),median(temperature),min(temperature),mode(temperature),spread(temperature),stddev(temperature),sum(temperature) from factory where ((workshop=\"A1\" and production=\"B1\" and cell =\"C1\" ) or temperature< 15 )", database); result = influxDB.query(query); System.out.println("query2 result:" + result.getResults().get(0).getSeries().get(0).toString()); diff --git a/influxdb-protocol/src/test/java/org/apache/iotdb/influxdb/integration/IoTDBInfluxDBIT.java b/influxdb-protocol/src/test/java/org/apache/iotdb/influxdb/integration/IoTDBInfluxDBIT.java index d8c97dc2b4ab..047cada0f145 100644 --- a/influxdb-protocol/src/test/java/org/apache/iotdb/influxdb/integration/IoTDBInfluxDBIT.java +++ b/influxdb-protocol/src/test/java/org/apache/iotdb/influxdb/integration/IoTDBInfluxDBIT.java @@ -33,11 +33,15 @@ import org.testcontainers.containers.GenericContainer; import org.testcontainers.utility.DockerImageName; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class IoTDBInfluxDBIT { @@ -164,8 +168,9 @@ public void testCommonQueryColumn() { QueryResult.Series series = result.getResults().get(0).getSeries().get(0); String[] retArray = new String[] {"time", "name", "sex", "province", "country", "score", "tel"}; + Set columnNames = new HashSet<>(Arrays.asList(retArray)); for (int i = 0; i < series.getColumns().size(); i++) { - assertEquals(retArray[i], series.getColumns().get(i)); + assertTrue(columnNames.contains(series.getColumns().get(i))); } } diff --git a/lsm/README.md b/lsm/README.md new file mode 100644 index 000000000000..d1c2ae75975f --- /dev/null +++ b/lsm/README.md @@ -0,0 +1,396 @@ + + +# LSM Document +
+  _____      ______   ____    ____  
+ |_   _|   .' ____ \ |_   \  /   _| 
+   | |     | (___ \_|  |   \/   |   
+   | |   _  _.____`.   | |\  /| |   
+  _| |__/ || \____) | _| |_\/_| |_  
+ |________| \______.'|_____||_____| > version 0.14.0-SNAPSHOT
+                                    
+
+## Abstract + +The lsm framework has implemented the **memory structure** at present, and users only need to define the memory structure and access method of each level. + + +## Example +Suppose we need to implement an lsm storage engine to store each record similar to , we define the following four layers of memory nodes. +### Memory structure +Implement the memory structure of each layer + +- MemChunk + +The last layer of memory nodes, save the id list +```java +public class MemChunk { + List deviceIDS; + + public MemChunk() { + deviceIDS = new ArrayList<>(); + } + + public List getDeviceIDS() { + return deviceIDS; + } + + public void setDeviceIDS(List deviceIDS) { + this.deviceIDS = deviceIDS; + } + + @Override + public String toString() { + return "MemChunk{" + deviceIDS.toString() + '}'; + } +} +``` + +- MemGroup + +Use a map to manage tagValue->MemChunk +```java +public class MemGroup { + + Map map; + + public MemGroup() { + map = new HashMap<>(); + } + + public Map getMap() { + return map; + } + + public void setMap(Map map) { + this.map = map; + } + + @Override + public String toString() { + return "MemGroup{" + map.toString() + '}'; + } +} +``` + +- MemTable + +Use a map to manage tagKey->MemGroup +```java +public class MemTable { + + private Map map; + + public MemTable() { + map = new HashMap<>(); + } + + public Map getMap() { + return map; + } + + public void setMap(Map map) { + this.map = map; + } + + @Override + public String toString() { + return "MemTable{" + map.toString() + '}'; + } +} +``` + +- MemTableManager + +Manage working memTable, and immutable memTable +```java +public class MemTableManager { + + private MemTable working; + + private Map immutables; + + private int maxDeviceID; + + public MemTableManager() { + working = new MemTable(); + immutables = new HashMap<>(); + maxDeviceID = 0; + } + + public MemTable getWorking() { + return working; + } + + public void setWorking(MemTable working) { + this.working = working; + } + + public Map getImmutables() { + return immutables; + } + + public void setImmutables(Map immutables) { + this.immutables = immutables; + } + + public int getMaxDeviceID() { + return maxDeviceID; + } + + public void setMaxDeviceID(int maxDeviceID) { + this.maxDeviceID = maxDeviceID; + } + + @Override + public String toString() { + return "MemTableManager{" + + "working=" + + working.toString() + + ", immutables=" + + immutables.toString() + + '}'; + } +} +``` + +### Access method +Incoming access to each layer for the framework + +- Insertion and flush example +```java +public class Main { + public static void main(String[] args) throws Exception { + MemTableManager memTableManager = new MemTableManager(); + System.out.println("-------------insert--------------"); + insertionExample(memTableManager); + System.out.println("-------------flush--------------"); + flushExample(memTableManager); + } + + public static void insertionExample(MemTableManager memTableManager) throws Exception { + // Initialize a BasicLsmManager to manage insert operations + BasicLsmManager baseLsmManager = + new BasicLsmManager(); + baseLsmManager + .nextLevel( + // The insert method of the MemTableManager level + new InsertLevelProcess() { + @Override + public List getChildren( + MemTableManager memNode, InsertRequestContext context) { + Integer deviceID = (Integer) context.getValue(); + int maxDeviceID = memNode.getMaxDeviceID(); + List children = new ArrayList<>(); + if (deviceID / 65536 == maxDeviceID / 65536) { + children.add(memNode.getWorking()); + } else { + children.add(memNode.getImmutables().get(deviceID / 65536)); + } + return children; + } + + @Override + public void insert(MemTableManager memNode, InsertRequestContext context) { + Integer deviceID = (Integer) context.getValue(); + int maxDeviceID = memNode.getMaxDeviceID(); + if (deviceID / 65536 == maxDeviceID / 65536) { + if (memNode.getWorking() == null) { + memNode.setWorking(new MemTable()); + } + } else if (deviceID > maxDeviceID) { + memNode + .getImmutables() + .put(memNode.getMaxDeviceID() / 65536, memNode.getWorking()); + memNode.setWorking(new MemTable()); + } + if (deviceID > maxDeviceID) { + memNode.setMaxDeviceID(deviceID); + } + } + }) + .nextLevel( + // The insert method of the MemTable level + new InsertLevelProcess() { + @Override + public List getChildren(MemTable memNode, InsertRequestContext context) { + String key = (String) context.getKey(); + List children = new ArrayList<>(); + children.add(memNode.getMap().get(key)); + return children; + } + + @Override + public void insert(MemTable memNode, InsertRequestContext context) { + String key = (String) context.getKey(); + Map map = memNode.getMap(); + if (map.containsKey(key)) return; + map.put(key, new MemGroup()); + } + }) + .nextLevel( + // The insert method of the MemGroup level + new InsertLevelProcess() { + @Override + public List getChildren(MemGroup memNode, InsertRequestContext context) { + String key = (String) context.getKey(); + List children = new ArrayList<>(); + children.add(memNode.getMap().get(key)); + return children; + } + + @Override + public void insert(MemGroup memNode, InsertRequestContext context) { + String key = (String) context.getKey(); + Map map = memNode.getMap(); + if (map.containsKey(key)) return; + map.put(key, new MemChunk()); + } + }) + .nextLevel( + // The insert method of the MemChunk level + new InsertLevelProcess() { + @Override + public List getChildren(MemChunk memNode, InsertRequestContext context) { + return null; + } + + @Override + public void insert(MemChunk memNode, InsertRequestContext context) { + Integer deviceID = (Integer) context.getValue(); + List deviceIDs = memNode.getDeviceIDS(); + deviceIDs.add(deviceID); + } + }); + + // Insert some records + // The key at the MemTableManager level is null + baseLsmManager.process(memTableManager, new InsertRequestContext(1, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(2, null, "a", "d")); + baseLsmManager.process(memTableManager, new InsertRequestContext(3, null, "a", "e")); + baseLsmManager.process(memTableManager, new InsertRequestContext(4, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(5, null, "a1", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(6, null, "a2", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(65535, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(65536, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(2, null, "a", "d")); + baseLsmManager.process(memTableManager, new InsertRequestContext(3, null, "a", "e")); + baseLsmManager.process(memTableManager, new InsertRequestContext(4, null, "a", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(5, null, "a1", "b")); + baseLsmManager.process(memTableManager, new InsertRequestContext(6, null, "a2", "b")); + // process memTableManager + System.out.println(memTableManager); + } + + public static void flushExample(MemTableManager memTableManager) throws Exception { + // Initialize a BasicLsmManager to manage insert operations + BasicLsmManager flushManager = + new BasicLsmManager(); + + flushManager + .nextLevel( + // The insert method of the MemTableManager level + new FlushLevelProcess() { + @Override + public void flush(MemTableManager memNode, FlushRequestContext context) { + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); + } + + @Override + public List getChildren( + MemTableManager memNode, FlushRequestContext context) { + List memTables = new ArrayList<>(); + memTables.addAll(memNode.getImmutables().values()); + if (memNode.getWorking() != null) memTables.add(memNode.getWorking()); + return memTables; + } + }) + .nextLevel( + // The insert method of the MemTable level + new FlushLevelProcess() { + @Override + public void flush(MemTable memNode, FlushRequestContext context) { + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); + } + + @Override + public List getChildren(MemTable memNode, FlushRequestContext context) { + List memGroups = new ArrayList<>(); + memGroups.addAll(memNode.getMap().values()); + return memGroups; + } + }) + .nextLevel( + // The insert method of the MemGroup level + new FlushLevelProcess() { + @Override + public void flush(MemGroup memNode, FlushRequestContext context) { + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); + } + + @Override + public List getChildren(MemGroup memNode, FlushRequestContext context) { + List memChunk = new ArrayList<>(); + memChunk.addAll(memNode.getMap().values()); + return memChunk; + } + }) + .nextLevel( + // The insert method of the MemChunk level + new FlushLevelProcess() { + @Override + public void flush(MemChunk memNode, FlushRequestContext context) { + System.out.println("FLUSH: " + memNode + "-->[level:" + context.getLevel() + "]"); + } + + @Override + public List getChildren(MemChunk memNode, FlushRequestContext context) { + return new ArrayList<>(); + } + }); + + // process memTableManager + flushManager.process(memTableManager, new FlushRequestContext()); + } +} + +``` +- Output + +```txt +-------------insert-------------- +MemTableManager{working=MemTable{{a=MemGroup{{b=MemChunk{[65536]}}}}}, immutables={0=MemTable{{a1=MemGroup{{b=MemChunk{[5, 5]}}}, a=MemGroup{{b=MemChunk{[1, 4, 65535, 4]}, d=MemChunk{[2, 2]}, e=MemChunk{[3, 3]}}}, a2=MemGroup{{b=MemChunk{[6, 6]}}}}}}} +-------------flush-------------- +FLUSH: MemChunk{[5, 5]}-->[level:3] +FLUSH: MemChunk{[1, 4, 65535, 4]}-->[level:3] +FLUSH: MemChunk{[2, 2]}-->[level:3] +FLUSH: MemChunk{[3, 3]}-->[level:3] +FLUSH: MemChunk{[6, 6]}-->[level:3] +FLUSH: MemChunk{[65536]}-->[level:3] +FLUSH: MemGroup{{b=MemChunk{[5, 5]}}}-->[level:2] +FLUSH: MemGroup{{b=MemChunk{[1, 4, 65535, 4]}, d=MemChunk{[2, 2]}, e=MemChunk{[3, 3]}}}-->[level:2] +FLUSH: MemGroup{{b=MemChunk{[6, 6]}}}-->[level:2] +FLUSH: MemGroup{{b=MemChunk{[65536]}}}-->[level:2] +FLUSH: MemTable{{a1=MemGroup{{b=MemChunk{[5, 5]}}}, a=MemGroup{{b=MemChunk{[1, 4, 65535, 4]}, d=MemChunk{[2, 2]}, e=MemChunk{[3, 3]}}}, a2=MemGroup{{b=MemChunk{[6, 6]}}}}}-->[level:1] +FLUSH: MemTable{{a=MemGroup{{b=MemChunk{[65536]}}}}}-->[level:1] +FLUSH: MemTableManager{working=MemTable{{a=MemGroup{{b=MemChunk{[65536]}}}}}, immutables={0=MemTable{{a1=MemGroup{{b=MemChunk{[5, 5]}}}, a=MemGroup{{b=MemChunk{[1, 4, 65535, 4]}, d=MemChunk{[2, 2]}, e=MemChunk{[3, 3]}}}, a2=MemGroup{{b=MemChunk{[6, 6]}}}}}}}-->[level:0] +``` diff --git a/lsm/pom.xml b/lsm/pom.xml new file mode 100644 index 000000000000..b95e98aea9d2 --- /dev/null +++ b/lsm/pom.xml @@ -0,0 +1,32 @@ + + + + + iotdb-parent + org.apache.iotdb + 0.14.0-SNAPSHOT + ../pom.xml + + 4.0.0 + iotdb-lsm + IoTDB lsm + diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java new file mode 100644 index 000000000000..548874625265 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/DeleteRequestContext.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import org.apache.iotdb.lsm.strategy.PostOrderAccessStrategy; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * represents the context of a deletion request, this class can be extended to implement a custom + * context + */ +public class DeleteRequestContext extends RequestContext { + + // save the key of each level + List keys; + + // value to delete + Object value; + + public DeleteRequestContext() { + super(); + type = RequestType.DELETE; + // post-order traversal strategy is used by default + accessStrategy = new PostOrderAccessStrategy(); + } + + public DeleteRequestContext(Object value, Object... ks) { + super(); + this.value = value; + keys = new ArrayList<>(); + keys.addAll(Arrays.asList(ks)); + type = RequestType.DELETE; + // post-order traversal strategy is used by default + accessStrategy = new PostOrderAccessStrategy(); + } + + public void setKeys(List keys) { + this.keys = keys; + } + + public void setValue(Object value) { + this.value = value; + } + + public Object getKey() { + return keys.get(level); + } + + public List getKeys() { + return keys; + } + + public Object getValue() { + return value; + } + + public int size() { + return keys.size(); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java new file mode 100644 index 000000000000..d2a9cd8bae10 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/FlushRequestContext.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import org.apache.iotdb.lsm.strategy.RBFSAccessStrategy; + +/** + * represents the context of a flush request, this class can be extended to implement a custom + * context + */ +public class FlushRequestContext extends RequestContext { + public FlushRequestContext() { + super(); + type = RequestType.FLUSH; + // use the reverse breadth-first traversal strategy to access memory nodes + accessStrategy = new RBFSAccessStrategy(); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java new file mode 100644 index 000000000000..933f006a58b0 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/InsertRequestContext.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import org.apache.iotdb.lsm.strategy.PreOrderAccessStrategy; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * represents the context of a insertion request, this class can be extended to implement a custom + * context + */ +public class InsertRequestContext extends RequestContext { + + // save the key of each level + List keys; + + // value to insert + Object value; + + public InsertRequestContext() { + super(); + type = RequestType.INSERT; + // preorder traversal strategy is used by default + accessStrategy = new PreOrderAccessStrategy(); + } + + public InsertRequestContext(Object value, Object... keys) { + super(); + this.value = value; + this.keys = new ArrayList<>(); + this.keys.addAll(Arrays.asList(keys)); + type = RequestType.INSERT; + // preorder traversal strategy is used by default + accessStrategy = new PreOrderAccessStrategy(); + } + + public Object getKey() { + return keys.get(level); + } + + public void setKeys(List keys) { + this.keys = keys; + } + + public void setValue(Object value) { + this.value = value; + } + + public List getKeys() { + return keys; + } + + public Object getValue() { + return value; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java new file mode 100644 index 000000000000..8742ac4e2ace --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/QueryRequestContext.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import org.apache.iotdb.lsm.strategy.PostOrderAccessStrategy; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * represents the context of a query request, this class can be extended to implement a custom + * context + */ +public class QueryRequestContext extends RequestContext { + + // save the key of each level + List keys; + + public QueryRequestContext(Object... ks) { + super(); + keys = new ArrayList<>(); + keys.addAll(Arrays.asList(ks)); + type = RequestType.QUERY; + // post-order traversal strategy is used by default + accessStrategy = new PostOrderAccessStrategy(); + } + + public Object getKey() { + return keys.get(level); + } + + public int size() { + return keys.size(); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java new file mode 100644 index 000000000000..4866be0ca1fd --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/RequestContext.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +import org.apache.iotdb.lsm.strategy.AccessStrategy; +import org.apache.iotdb.lsm.strategy.PreOrderAccessStrategy; + +/** represents the context of a request */ +public class RequestContext { + + // request type + RequestType type; + + // memory Structure Access Policy + AccessStrategy accessStrategy; + + // the tree level of the currently pending memory node + int level; + + // the maximum level of memory nodes that can be processed + int levelUpperBound; + + // return value after request processing is complete + Object result; + + // whether the request context is only used for recovery + boolean recover; + + public RequestContext() { + // preorder traversal strategy is used by default + accessStrategy = new PreOrderAccessStrategy(); + type = RequestType.NONE; + level = 0; + levelUpperBound = Integer.MAX_VALUE; + recover = false; + } + + public void setLevel(int level) { + this.level = level; + } + + public int getLevel() { + return level; + } + + public RequestType getType() { + return type; + } + + public void setType(RequestType type) { + this.type = type; + } + + public Object getResult() { + return result; + } + + public void setResult(Object result) { + this.result = result; + } + + public AccessStrategy getAccessStrategy() { + return accessStrategy; + } + + public void setAccessStrategy(AccessStrategy accessStrategy) { + this.accessStrategy = accessStrategy; + } + + public int getLevelUpperBound() { + return levelUpperBound; + } + + public void setLevelUpperBound(int levelUpperBound) { + this.levelUpperBound = levelUpperBound; + } + + public boolean isRecover() { + return recover; + } + + public void setRecover(boolean recover) { + this.recover = recover; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/context/RequestType.java b/lsm/src/main/java/org/apache/iotdb/lsm/context/RequestType.java new file mode 100644 index 000000000000..e7dfac6c2a9e --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/context/RequestType.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context; + +public enum RequestType { + NONE, + INSERT, + QUERY, + DELETE, + FLUSH; +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java new file mode 100644 index 000000000000..4e10ba794bdc --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcess.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.RequestContext; + +import java.util.List; + +/** the processing method corresponding to each layer of memory nodes */ +public abstract class BasicLevelProcess + implements LevelProcess { + + // the next level process + LevelProcess next; + + /** + * process the current layer memory node + * + * @param memNode memory node + * @param context request context + */ + public abstract void handle(I memNode, C context); + + /** + * get the memory node that needs to be processed in the next layer + * + * @param memNode memory node + * @param context request context + * @return all next-level memory nodes that need to be processed + */ + public abstract List getChildren(I memNode, C context); + + /** + * add the LevelProcess of the next layer of memory nodes + * + * @param next LevelProcess of the next layer + * @return the next level process + */ + @Override + public LevelProcess nextLevel(LevelProcess next) { + this.next = next; + return next; + } + + /** + * use this method to process memory nodes at each layer according to the access strategy + * + * @param memNode memory node + * @param context request context + */ + @Override + public void process(I memNode, C context) { + context.getAccessStrategy().execute(this, memNode, context); + } + + public boolean hasNext() { + return next != null; + } + + public LevelProcess getNext() { + return next; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java new file mode 100644 index 000000000000..bd71d7fc8679 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcess.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.DeleteRequestContext; + +/** indicates the deletion method of each layer of memory nodes */ +public abstract class DeleteLevelProcess + extends BasicLevelProcess { + + /** + * the deletion method of memory node + * + * @param memNode memory node + * @param context deletion request context + */ + public abstract void delete(I memNode, DeleteRequestContext context); + + @Override + public void handle(I memNode, DeleteRequestContext context) { + delete(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java new file mode 100644 index 000000000000..46d72b4b5b50 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcess.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.FlushRequestContext; + +/** indicates the flush method of each layer of memory nodes */ +public abstract class FlushLevelProcess extends BasicLevelProcess { + + /** + * the flush method of memory node + * + * @param memNode memory node + * @param context flush request context + */ + public abstract void flush(I memNode, FlushRequestContext context); + + public void handle(I memNode, FlushRequestContext context) { + flush(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java new file mode 100644 index 000000000000..cf7d9c69bb77 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcess.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.InsertRequestContext; + +/** indicates the insertion method of each layer of memory nodes */ +public abstract class InsertLevelProcess + extends BasicLevelProcess { + + /** + * the insertion method of memory node + * + * @param memNode memory node + * @param context insertion request context + */ + public abstract void insert(I memNode, InsertRequestContext context); + + @Override + public void handle(I memNode, InsertRequestContext context) { + insert(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java new file mode 100644 index 000000000000..763d6eb4f6d3 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcess.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.RequestContext; + +/** the processing method corresponding to each layer of memory nodes */ +public interface LevelProcess { + + /** + * add the LevelProcess of the next layer of memory nodes + * + * @param next LevelProcess of the next layer + * @return LevelProcess of the next layer + */ + LevelProcess nextLevel(LevelProcess next); + + /** + * use this method to process memory nodes at each layer according to the access strategy + * + * @param memNode memory node + * @param context request context + */ + void process(I memNode, C context); +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java new file mode 100644 index 000000000000..2fef2f213d05 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcess.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.QueryRequestContext; + +/** indicates the query method of each layer of memory nodes */ +public abstract class QueryLevelProcess extends BasicLevelProcess { + + /** + * the query method of memory node + * + * @param memNode memory node + * @param context query request context + */ + public abstract void query(I memNode, QueryRequestContext context); + + @Override + public void handle(I memNode, QueryRequestContext context) { + query(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java new file mode 100644 index 000000000000..5b8816f7f2f2 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/BasicLsmManager.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.manager; + +import org.apache.iotdb.lsm.context.RequestContext; +import org.apache.iotdb.lsm.levelProcess.LevelProcess; + +/** */ +public class BasicLsmManager implements LsmManager { + + // the level process of the first layer of memory nodes + LevelProcess levelProcess; + + /** + * preprocessing of the root memory node + * + * @param root root memory node + * @param context request context + * @throws Exception + */ + public void preProcess(T root, C context) throws Exception {} + + /** + * postprocessing of the root memory node + * + * @param root root memory node + * @param context request context + * @throws Exception + */ + public void postProcess(T root, C context) throws Exception {} + + /** + * processing of the root memory node + * + * @param root root memory node + * @param context request context + * @throws Exception + */ + @Override + public void process(T root, C context) throws Exception { + preProcess(root, context); + levelProcess.process(root, context); + postProcess(root, context); + } + + /** + * add the LevelProcess of the next layer of memory nodes + * + * @param levelProcess LevelProcess of the next layer + * @return LevelProcess of the next layer + */ + @Override + public LevelProcess nextLevel(LevelProcess levelProcess) { + this.levelProcess = levelProcess; + return levelProcess; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java new file mode 100644 index 000000000000..ea2355378e5d --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/manager/LsmManager.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.manager; + +import org.apache.iotdb.lsm.context.RequestContext; +import org.apache.iotdb.lsm.levelProcess.LevelProcess; + +// used to implement lsm manager +public interface LsmManager { + + /** + * use this method to process root memory node + * + * @param memNode memory node + * @param context request context + */ + void process(T memNode, C context) throws Exception; + + /** + * add the LevelProcess of the next layer of memory nodes + * + * @param next LevelProcess of the next layer + * @return LevelProcess of the next layer + */ + LevelProcess nextLevel(LevelProcess next); +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java new file mode 100644 index 000000000000..567f83faa3e3 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/AccessStrategy.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.RequestContext; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +/** access strategy for memory nodes */ +public interface AccessStrategy { + + /** + * implementation of access strategy + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ + void execute( + BasicLevelProcess levelProcess, I memNode, C context); +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java new file mode 100644 index 000000000000..e93bd5ca959e --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.RequestContext; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; + +/** breadth-first access strategy implementation class */ +public class BFSAccessStrategy implements AccessStrategy { + + // same level memory nodes, used to implement BFSAccessStrategy + Queue sameLevelMemNodes; + + /** + * breadth-first access strategy implementation + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ + @Override + public void execute( + BasicLevelProcess levelProcess, I memNode, C context) { + List children = new ArrayList<>(); + int currentLevel = context.getLevel(); + if (sameLevelMemNodes == null) { + sameLevelMemNodes = new LinkedList<>(); + // process the current memory node + levelProcess.handle(memNode, context); + // get all memory nodes to be processed in the next layer + children = levelProcess.getChildren(memNode, context); + } else { + while (!sameLevelMemNodes.isEmpty()) { + I node = (I) sameLevelMemNodes.poll(); + levelProcess.handle(node, context); + children.addAll(levelProcess.getChildren(node, context)); + } + } + sameLevelMemNodes.addAll(children); + context.setLevel(currentLevel + 1); + if (levelProcess.hasNext() && !sameLevelMemNodes.isEmpty()) { + levelProcess.getNext().process(null, context); + } + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java new file mode 100644 index 000000000000..5e96f29b1b45 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.RequestContext; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +import java.util.List; + +/** post-order traversal access strategy implementation class */ +public class PostOrderAccessStrategy implements AccessStrategy { + + /** + * post-order traversal access strategy + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ + @Override + public void execute( + BasicLevelProcess levelProcess, I memNode, C context) { + int currentLevel = context.getLevel(); + AccessStrategy accessStrategy = context.getAccessStrategy(); + // get all memory nodes to be processed in the next layer + List children = levelProcess.getChildren(memNode, context); + if (levelProcess.hasNext()) { + context.setLevel(currentLevel + 1); + for (O child : children) { + // process next level memory node + levelProcess.getNext().process(child, context); + } + } + + context.setLevel(currentLevel); + context.setAccessStrategy(accessStrategy); + // process the current memory node + levelProcess.handle(memNode, context); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java new file mode 100644 index 000000000000..d16780cf59e5 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.RequestContext; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +import java.util.List; + +/** pre-order traversal access strategy implementation class */ +public class PreOrderAccessStrategy implements AccessStrategy { + + /** + * pre-order traversal access strategy + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ + @Override + public void execute( + BasicLevelProcess levelProcess, I memNode, C context) { + int currentLevel = context.getLevel(); + // process the current memory node + levelProcess.handle(memNode, context); + // get all memory nodes to be processed in the next layer + List children = levelProcess.getChildren(memNode, context); + + if (levelProcess.hasNext()) { + context.setLevel(currentLevel + 1); + for (O child : children) { + // process next level memory node + levelProcess.getNext().process(child, context); + } + } + context.setLevel(currentLevel); + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java new file mode 100644 index 000000000000..13b05c51717a --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.RequestContext; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcess; + +import java.util.List; + +/** reverse breadth first traversal access strategy implementation class */ +public class RBFSAccessStrategy implements AccessStrategy { + + /** + * reverse breadth first traversal access strategy + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ + @Override + public void execute( + BasicLevelProcess levelProcess, I memNode, C context) { + int currentLevel = context.getLevel(); + + // if the upper bound has not been set and there is no next-level processing method, set the + // upper bound to the current level + if (Integer.MAX_VALUE == context.getLevelUpperBound() && !levelProcess.hasNext()) { + context.setLevelUpperBound(context.getLevel()); + } + + // if the current memory node is the root + if (currentLevel == 0) { + // if all the next level nodes of the root node have not been processed + while (context.getLevelUpperBound() != currentLevel) { + // process all pending next-level nodes + List children = levelProcess.getChildren(memNode, context); + for (O child : children) { + context.setLevel(currentLevel + 1); + // use the processing method of the next layer to process the next layer of nodes + levelProcess.getNext().process(child, context); + context.setLevel(currentLevel); + } + + // after each layer is processed, the upper bound is reduced by one + context.setLevelUpperBound(context.getLevelUpperBound() - 1); + } + + // process the current memory node + levelProcess.handle(memNode, context); + return; + } + + if (currentLevel > context.getLevelUpperBound()) return; + + // only process memory nodes with equal level and upper bound + if (currentLevel == context.getLevelUpperBound()) { + levelProcess.handle(memNode, context); + return; + } + + // process all pending next-level nodes + List children = levelProcess.getChildren(memNode, context); + for (O child : children) { + context.setLevel(currentLevel + 1); + levelProcess.getNext().process(child, context); + context.setLevel(currentLevel); + } + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java new file mode 100644 index 000000000000..c1e42f74113b --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import java.io.FileNotFoundException; +import java.io.IOException; + +/** get records in wal file */ +public interface IWALReader { + + /** + * close resource + * + * @throws IOException + */ + void close() throws IOException; + + /** + * determine if there is a next record + * + * @return returns true if there is, else returns false + * @throws FileNotFoundException + */ + boolean hasNext() throws FileNotFoundException; + + /** + * return the next record + * + * @throws FileNotFoundException + */ + WALRecord next() throws FileNotFoundException; +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java new file mode 100644 index 000000000000..3a7b4e2cda85 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import java.io.IOException; + +/** write records to wal file */ +public interface IWALWriter { + + /** + * write walRecord to wal file + * + * @param walRecord record to be written + * @throws IOException + */ + void write(WALRecord walRecord) throws IOException; + + /** + * force brush + * + * @throws IOException + */ + void force() throws IOException; + + /** + * close resource + * + * @throws IOException + */ + void close() throws IOException; +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java new file mode 100644 index 000000000000..a5a70b62f172 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.NoSuchElementException; + +/** get records in wal file */ +public class WALReader implements IWALReader { + private static final Logger logger = LoggerFactory.getLogger(WALReader.class); + // wal file + private final File logFile; + // wal record prototype, clone on read + private final WALRecord prototype; + private DataInputStream logStream; + // next wal record + private WALRecord nextRecord; + private boolean fileCorrupted = false; + + public WALReader(File logFile, WALRecord prototype) throws IOException { + this.logFile = logFile; + this.logStream = + new DataInputStream(new BufferedInputStream(Files.newInputStream(logFile.toPath()))); + this.prototype = prototype; + } + + @Override + public void close() throws IOException { + logStream.close(); + logStream = null; + } + + @Override + public boolean hasNext() { + if (nextRecord != null) { + return true; + } + try { + if (fileCorrupted) { + return false; + } + int logSize = logStream.readInt(); + if (logSize <= 0) { + return false; + } + // first clone the object through the prototype + nextRecord = prototype.clone(); + // then perform deserialization and assign a value to the new object + nextRecord.deserialize(logStream); + } catch (EOFException e) { + logger.info(e.getMessage()); + return false; + } catch (IOException e) { + logger.warn(e.getMessage()); + fileCorrupted = true; + return false; + } + return true; + } + + @Override + public WALRecord next() { + if (nextRecord == null) { + throw new NoSuchElementException(); + } + WALRecord walRecord = nextRecord; + nextRecord = null; + return walRecord; + } + + @Override + public String toString() { + return "WALReader{" + "logFile=" + logFile + '}'; + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java new file mode 100644 index 000000000000..0aaf77460aab --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALRecord.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +/** represents a wal record, which can be extended to implement more complex wal records */ +public abstract class WALRecord implements Cloneable { + + /** + * serialize the wal record + * + * @param buffer byte buffer + */ + public abstract void serialize(ByteBuffer buffer); + + /** + * deserialize via input stream + * + * @param stream data input stream + * @throws IOException + */ + public abstract void deserialize(DataInputStream stream) throws IOException; + + @Override + public WALRecord clone() { + try { + return (WALRecord) super.clone(); + } catch (CloneNotSupportedException e) { + throw new AssertionError(e.getMessage()); + } + } +} diff --git a/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java new file mode 100644 index 000000000000..5ce97779ce12 --- /dev/null +++ b/lsm/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.FileChannel; + +/** write records to wal file */ +public class WALWriter implements IWALWriter { + private static final Logger logger = LoggerFactory.getLogger(WALWriter.class); + // wal file + private File logFile; + private FileOutputStream fileOutputStream; + private FileChannel channel; + // 4-bit buffer + private final ByteBuffer lengthBuffer; + // save wal record serialized byte data + private final ByteBuffer walBuffer; + private final boolean forceEachWrite; + + public WALWriter(File logFile, int walBufferSize, boolean forceEachWrite) + throws FileNotFoundException { + this.logFile = logFile; + this.forceEachWrite = forceEachWrite; + fileOutputStream = new FileOutputStream(logFile, true); + channel = fileOutputStream.getChannel(); + lengthBuffer = ByteBuffer.allocate(4); + walBuffer = ByteBuffer.allocate(walBufferSize); + } + + /** + * write walRecord to wal file + * + * @param walRecord record to be written + * @throws IOException + */ + @Override + public void write(WALRecord walRecord) throws IOException { + if (channel == null) { + fileOutputStream = new FileOutputStream(logFile, true); + channel = fileOutputStream.getChannel(); + } + walBuffer.clear(); + walRecord.serialize(walBuffer); + walBuffer.flip(); + int logSize = walBuffer.limit(); + lengthBuffer.clear(); + lengthBuffer.putInt(logSize); + lengthBuffer.flip(); + + try { + channel.write(lengthBuffer); + channel.write(walBuffer); + + if (this.forceEachWrite) { + channel.force(true); + } + } catch (ClosedChannelException ignored) { + logger.warn("someone interrupt current thread, so no need to do write for io safety"); + } + } + + @Override + public void force() throws IOException { + if (channel != null && channel.isOpen()) { + channel.force(true); + } + } + + @Override + public void close() throws IOException { + if (channel != null) { + if (channel.isOpen()) { + channel.force(true); + } + fileOutputStream.close(); + fileOutputStream = null; + channel.close(); + channel = null; + } + } + + @Override + public String toString() { + return "WALLogWriter{" + "logFile=" + logFile + '}'; + } +} diff --git a/pom.xml b/pom.xml index a7f5012ec118..6bb952ace414 100644 --- a/pom.xml +++ b/pom.xml @@ -121,6 +121,8 @@ trigger-api rewrite-tsfile-tool external-api + lsm + schema-engine-tag diff --git a/schema-engine-tag/README.md b/schema-engine-tag/README.md new file mode 100644 index 000000000000..e4d70f9e1502 --- /dev/null +++ b/schema-engine-tag/README.md @@ -0,0 +1,190 @@ + + +# Tag Schema Region +`TagSchemaRegion` is an implementation of `SchemaRegion` +
+  _____             _____      _                           ______           _             
+|_   _|           /  ___|    | |                          | ___ \         (_)            
+  | | __ _  __ _  \ `--.  ___| |__   ___ _ __ ___   __ _  | |_/ /___  __ _ _  ___  _ __  
+  | |/ _` |/ _` |  `--. \/ __| '_ \ / _ \ '_ ` _ \ / _` | |    // _ \/ _` | |/ _ \| '_ \ 
+  | | (_| | (_| | /\__/ / (__| | | |  __/ | | | | | (_| | | |\ \  __/ (_| | | (_) | | | |
+  \_/\__,_|\__, | \____/ \___|_| |_|\___|_| |_| |_|\__,_| \_| \_\___|\__, |_|\___/|_| |_|
+            __/ |                                                     __/ |              
+           |___/                                                     |___/ > version 0.14.0-SNAPSHOT
+
+ +# How To Use + +Firstly, you should package **schema-engine-tag** by the following command: + +```shell +mvn clean package -pl schema-engine-tag -am -DskipTests +``` + +After that, you can get a **conf** directory and a **lib** directory in +schema-engine-tag/target/schema-engine-tag. Copy the file in the conf directory to the conf directory of server, +and copy the files in the lib directory to the lib directory of server. + +Then, open the **iotdb-datanode.properties** in the conf directory of server, and set the `schema_engine_mode` to +**Tag**, set the `enable_id_table` to **true**. Restart the IoTDB, the system will use `TagSchemaRegion` to manage +the metadata. + +## Use Cli + +IoTDB offers different ways to interact with server, here we introduce the basic steps of using Cli tool to insert and query data. +The command line cli is interactive, so you should see the welcome logo and statements if everything is ready: +```sql +--------------------- +Starting IoTDB Cli +--------------------- + _____ _________ ______ ______ +|_ _| | _ _ ||_ _ `.|_ _ \ + | | .--.|_/ | | \_| | | `. \ | |_) | + | | / .'`\ \ | | | | | | | __'. + _| |_| \__. | _| |_ _| |_.' /_| |__) | +|_____|'.__.' |_____| |______.'|_______/ version 0.14.0-SNAPSHOT + + +IoTDB> login successfully +``` +### create timeseries + +- create timeseries + +```sql +IoTDB> create timeseries root.ln.tag1.a.tag2.b.status with datatype=BOOLEAN,encoding=PLAIN +Msg: The statement is executed successfully. +``` +- create aligned timeseries + +```sql +IoTDB> CREATE ALIGNED TIMESERIES root.ln.tag1.a.tag2.c(latitude FLOAT encoding=PLAIN compressor=SNAPPY, longitude FLOAT encoding=PLAIN compressor=SNAPPY) + +Msg: The statement is executed successfully. +``` + +### show timeserie + +- point query + +enter a full path + +```sql +IoTDB> show timeseries root.ln.tag2.c.tag1.a ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| timeseries|alias|storage group|dataType|encoding|compression|tags|attributes| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| root.ln.tag1.a.tag2.c.latitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| +|root.ln.tag1.a.tag2.c.longitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +``` + +- batch query + +paths ending in ".**" indicate batch query + +```sql +IoTDB> show timeseries root.ln.tag1.a.** ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| timeseries|alias|storage group|dataType|encoding|compression|tags|attributes| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| root.ln.tag1.a.tag2.b.status| null| root.ln| BOOLEAN| PLAIN| SNAPPY|null| null| +| root.ln.tag1.a.tag2.c.latitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| +|root.ln.tag1.a.tag2.c.longitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ + +IoTDB> show timeseries root.ln.tag2.c.** ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| timeseries|alias|storage group|dataType|encoding|compression|tags|attributes| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| root.ln.tag1.a.tag2.c.latitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| +|root.ln.tag1.a.tag2.c.longitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ + +IoTDB> show timeseries root.ln.tag2.b.** ++----------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| timeseries|alias|storage group|dataType|encoding|compression|tags|attributes| ++----------------------------+-----+-------------+--------+--------+-----------+----+----------+ +|root.ln.tag1.a.tag2.b.status| null| root.ln| BOOLEAN| PLAIN| SNAPPY|null| null| ++----------------------------+-----+-------------+--------+--------+-----------+----+----------+ +``` + +### insert + +- insert a single column of data + +```sql +IoTDB> insert into root.ln.tag2.d(timestamp,status) values(1,true) +Msg: The statement is executed successfully. +IoTDB> insert into root.ln.tag2.d(timestamp,status) values(2,false) +Msg: The statement is executed successfully. +IoTDB> insert into root.ln.tag2.d(timestamp,status) values(3,true) +Msg: The statement is executed successfully. +IoTDB> insert into root.ln.tag1.a.tag2.d(timestamp,status) values(1,true) +Msg: The statement is executed successfully. +``` + +- insert alignment data + +```sql +IoTDB> insert into root.sg1.tag1.a(time, s1, s2) aligned values(2, 2, 2), (3, 3, 3) +Msg: The statement is executed successfully. +``` + +### select + +- point query + +```sql +IoTDB> select * from root.sg1.tag1.a ++-----------------------------+------------------+------------------+ +| Time|root.sg1.tag1.a.s1|root.sg1.tag1.a.s2| ++-----------------------------+------------------+------------------+ +|1970-01-01T08:00:00.002+08:00| 2.0| 2.0| +|1970-01-01T08:00:00.003+08:00| 3.0| 3.0| ++-----------------------------+------------------+------------------+ +``` + +- align by device + +```sql +IoTDB> select * from root.sg1.tag1.a align by device ++-----------------------------+---------------+---+---+ +| Time| Device| s1| s2| ++-----------------------------+---------------+---+---+ +|1970-01-01T08:00:00.002+08:00|root.sg1.tag1.a|2.0|2.0| +|1970-01-01T08:00:00.003+08:00|root.sg1.tag1.a|3.0|3.0| ++-----------------------------+---------------+---+---+ +``` + +- batch query + +```sql +IoTDB> select status from root.ln.tag2.d.** where time < 2017-11-01T00:08:00.000 ++-----------------------------+----------------------------+---------------------+ +| Time|root.ln.tag1.a.tag2.d.status|root.ln.tag2.d.status| ++-----------------------------+----------------------------+---------------------+ +|1970-01-01T08:00:00.001+08:00| true| true| +|1970-01-01T08:00:00.002+08:00| null| false| +|1970-01-01T08:00:00.003+08:00| null| true| ++-----------------------------+----------------------------+---------------------+ +``` \ No newline at end of file diff --git a/schema-engine-tag/pom.xml b/schema-engine-tag/pom.xml new file mode 100644 index 000000000000..184921f09895 --- /dev/null +++ b/schema-engine-tag/pom.xml @@ -0,0 +1,81 @@ + + + + + iotdb-parent + org.apache.iotdb + 0.14.0-SNAPSHOT + + 4.0.0 + schema-engine-tag + schema-engine-tag + + + org.roaringbitmap + RoaringBitmap + 0.9.32 + + + org.apache.iotdb + iotdb-lsm + ${project.version} + + + org.apache.iotdb + iotdb-server + ${project.version} + provided + + + + schema-engine-tag + + + org.apache.maven.plugins + maven-assembly-plugin + ${maven.assembly.version} + + + + schema-engine-tag-assembly + package + + single + + + + src/assembly/schema-engine-tag.xml + + false + + + true + true + + + + + + + + + diff --git a/schema-engine-tag/src/assembly/resources/conf/schema-tag.properties b/schema-engine-tag/src/assembly/resources/conf/schema-tag.properties new file mode 100644 index 000000000000..bfb7df48b8ab --- /dev/null +++ b/schema-engine-tag/src/assembly/resources/conf/schema-tag.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +#################### +### tag schema region Configuration +#################### + +# This configuration takes effect only when the schema engine mode is Tag. +# The mode is configured in the 'iotdb-datanode.properties'(schema_engine_mode=Tag). + +# Datatype: int +# The size of wal buffer used to store a wal record.(unit: byte) +# wal_buffer_size = 1024*1024 + +# Datatype: int +# How many device ids a memtable can insert, beyond which the memtable will become immutable +# num_of_deviceIds_in_memTable = 65536 \ No newline at end of file diff --git a/schema-engine-tag/src/assembly/schema-engine-tag.xml b/schema-engine-tag/src/assembly/schema-engine-tag.xml new file mode 100644 index 000000000000..49647f46a7a1 --- /dev/null +++ b/schema-engine-tag/src/assembly/schema-engine-tag.xml @@ -0,0 +1,45 @@ + + + + schema-engine-tag + + dir + zip + + false + + + /lib/tag-schema-region + + org.apache.iotdb:iotdb-lsm + org.roaringbitmap:RoaringBitmap + org.apache.iotdb:schema-engine-tag + + + + + + src/assembly/resources + ${file.separator} + + + diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java new file mode 100644 index 000000000000..d360a8432707 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/MockTagSchemaRegion.java @@ -0,0 +1,983 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +import org.apache.iotdb.common.rpc.thrift.TSchemaNode; +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.AlignedTimeseriesException; +import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; +import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.metadata.LocalSchemaProcessor; +import org.apache.iotdb.db.metadata.idtable.IDTable; +import org.apache.iotdb.db.metadata.idtable.IDTableManager; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.InsertMeasurementMNode; +import org.apache.iotdb.db.metadata.idtable.entry.SHA256DeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.metadata.mnode.EntityMNode; +import org.apache.iotdb.db.metadata.mnode.IMNode; +import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; +import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; +import org.apache.iotdb.db.metadata.path.MeasurementPath; +import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.metadata.schemaregion.SchemaRegionUtils; +import org.apache.iotdb.db.metadata.template.Template; +import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.mpp.common.schematree.MeasurementSchemaInfo; +import org.apache.iotdb.db.qp.physical.crud.InsertPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan; +import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan; +import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan; +import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.dataset.ShowDevicesResult; +import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; +import org.apache.iotdb.external.api.ISeriesNumerLimiter; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; + +/** Mock tag schema region, only used for test */ +public class MockTagSchemaRegion implements ISchemaRegion { + + protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private final String TAIL = ".**"; + private final IStorageGroupMNode storageGroupMNode; + private String storageGroupFullPath; + private SchemaRegionId schemaRegionId; + + private Map>> tagInvertedIndex; + + private List deviceIDS; + + private IDTable idTable; + + private final ISeriesNumerLimiter seriesNumerLimiter; + + public MockTagSchemaRegion( + PartialPath storageGroup, + SchemaRegionId schemaRegionId, + IStorageGroupMNode storageGroupMNode, + ISeriesNumerLimiter seriesNumerLimiter) + throws MetadataException { + + storageGroupFullPath = storageGroup.getFullPath(); + this.schemaRegionId = schemaRegionId; + this.storageGroupMNode = storageGroupMNode; + this.deviceIDS = new ArrayList<>(); + this.seriesNumerLimiter = seriesNumerLimiter; + tagInvertedIndex = new ConcurrentHashMap<>(); + idTable = IDTableManager.getInstance().getIDTable(storageGroup); + init(); + } + + @NotNull + private Map pathToTags(String path) { + if (path.length() <= storageGroupFullPath.length()) return new TreeMap<>(); + String devicePath = path.substring(storageGroupFullPath.length() + 1); + String[] tags = devicePath.split("\\."); + Map tagsMap = new TreeMap<>(); + for (int i = 0; i < tags.length; i += 2) { + tagsMap.put(tags[i], tags[i + 1]); + } + return tagsMap; + } + + public String tagsToPath(Map tags) { + StringBuilder stringBuilder = new StringBuilder(storageGroupFullPath); + for (String tagKey : tags.keySet()) { + stringBuilder.append(".").append(tagKey).append(".").append(tags.get(tagKey)); + } + return stringBuilder.toString(); + } + + @Override + public void init() throws MetadataException { + if (!config.isEnableIDTableLogFile() + && config.getDeviceIDTransformationMethod().equals("SHA256")) { + throw new MetadataException( + "enableIDTableLogFile OR deviceIDTransformationMethod==\"Plain\""); + } + } + + @Override + public void clear() { + return; + } + + @Override + public void forceMlog() { + return; + } + + @Override + public SchemaRegionId getSchemaRegionId() { + return schemaRegionId; + } + + @Override + public String getStorageGroupFullPath() { + return storageGroupFullPath; + } + + @Override + public void deleteSchemaRegion() throws MetadataException { + return; + } + + @Override + public boolean createSnapshot(File snapshotDir) { + return false; + } + + @Override + public void loadSnapshot(File latestSnapshotRootDir) { + return; + } + + private void createTagInvertedIndex(PartialPath devicePath) { + IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath); + Map tagsMap = pathToTags(devicePath.getFullPath()); + + deviceIDS.add(deviceID); + + for (String tagkey : tagsMap.keySet()) { + String tagValue = tagsMap.get(tagkey); + Map> tagkeyMap = + tagInvertedIndex.computeIfAbsent(tagkey, key -> new HashMap<>()); + List ids = tagkeyMap.computeIfAbsent(tagValue, key -> new ArrayList<>()); + ids.add(deviceIDS.size() - 1); + } + } + + private void createTimeseries( + PartialPath path, + TSDataType dataType, + TSEncoding encoding, + CompressionType compressor, + Map props) + throws MetadataException { + createTimeseries( + new CreateTimeSeriesPlan(path, dataType, encoding, compressor, props, null, null, null), 0); + } + + private void createAlignedTimeSeries( + PartialPath prefixPath, + List measurements, + List dataTypes, + List encodings, + List compressors) + throws MetadataException { + createAlignedTimeSeries( + new CreateAlignedTimeSeriesPlan( + prefixPath, measurements, dataTypes, encodings, compressors, null, null, null)); + } + + @Override + public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException { + PartialPath devicePath = plan.getPath().getDevicePath(); + Map tags = pathToTags(devicePath.getFullPath()); + PartialPath path = new PartialPath(tagsToPath(tags) + "." + plan.getPath().getMeasurement()); + plan.setPath(path); + devicePath = plan.getPath().getDevicePath(); + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is not aligned, please use createTimeseries or change entity.", + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } else if (deviceEntry.getMeasurementMap().containsKey(plan.getPath().getMeasurement())) { + throw new PathAlreadyExistException( + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } + } + idTable.createTimeseries(plan); + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + @Override + public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException { + PartialPath devicePath = plan.getPrefixPath(); + Map tags = pathToTags(devicePath.getFullPath()); + PartialPath path = new PartialPath(tagsToPath(tags)); + plan.setPrefixPath(path); + devicePath = plan.getPrefixPath(); + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (!deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is aligned, please use createAlignedTimeseries or change entity.", + devicePath.getFullPath()); + } else { + List measurements = plan.getMeasurements(); + List dataTypes = plan.getDataTypes(); + List encodings = plan.getEncodings(); + List compressors = plan.getCompressors(); + + List tmpMeasurements = new LinkedList<>(); + List tmpDataTypes = new LinkedList<>(); + List tmpEncodings = new LinkedList<>(); + List tmpCompressors = new LinkedList<>(); + for (int i = 0; i < measurements.size(); i++) { + String measurement = measurements.get(i); + if (!deviceEntry.getMeasurementMap().containsKey(measurement)) { + tmpMeasurements.add(measurements.get(i)); + tmpDataTypes.add(dataTypes.get(i)); + tmpEncodings.add(encodings.get(i)); + tmpCompressors.add(compressors.get(i)); + } + } + if (tmpMeasurements.size() == 0) + throw new PathAlreadyExistException(devicePath.getFullPath()); + plan.setMeasurements(tmpMeasurements); + plan.setDataTypes(tmpDataTypes); + plan.setEncodings(tmpEncodings); + plan.setCompressors(tmpCompressors); + } + } + idTable.createAlignedTimeseries(plan); + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + @Override + public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public int constructSchemaBlackList(PathPatternTree patternTree) throws MetadataException { + return 0; + } + + @Override + public void rollbackSchemaBlackList(PathPatternTree patternTree) throws MetadataException {} + + @Override + public List fetchSchemaBlackList(PathPatternTree patternTree) + throws MetadataException { + return null; + } + + @Override + public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException {} + + @Override + public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public boolean isPathExist(PartialPath path) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + int res = 0; + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + for (IDeviceID deviceID : deviceIDs) { + res += idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap().keySet().size(); + } + return res; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) + throws MetadataException { + return 0; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) + throws MetadataException { + return 0; + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, + int level, + boolean isPrefixMatch, + String key, + String value, + boolean isContains) + throws MetadataException { + return null; + } + + @Override + public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + if (pathPattern.getFullPath().length() <= storageGroupFullPath.length()) { + return deviceIDS.size(); + } else { + return getDeviceIDsByInvertedIndex(pathPattern).size(); + } + } + + @Override + public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public List getNodesListInGivenLevel( + PartialPath pathPattern, + int nodeLevel, + boolean isPrefixMatch, + LocalSchemaProcessor.StorageGroupFilter filter) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getChildNodePathInNextLevel(PartialPath pathPattern) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + Set res = new HashSet<>(); + String devicePath = pathPattern.getFullPath(); + if (!devicePath.endsWith(TAIL) && !devicePath.equals(storageGroupFullPath)) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry != null) { + res.add(pathPattern); + } + return res; + } + for (IDeviceID deviceID : deviceIDs) { + if (deviceID instanceof SHA256DeviceID) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map map = deviceEntry.getMeasurementMap(); + for (String m : map.keySet()) { + SchemaEntry schemaEntry = map.get(m); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + res.add( + new PartialPath( + diskSchemaEntry.seriesKey.substring( + 0, + diskSchemaEntry.seriesKey.length() + - diskSchemaEntry.measurementName.length() + - 1))); + break; + } + } else { + res.add(new PartialPath(deviceID.toStringID())); + } + } + return res; + } + + @Override + public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + PartialPath devicePath = pathPattern.getDevicePath(); + if (devicePath.getFullPath().endsWith(TAIL)) { + return getMeasurementPathsWithBatchQuery(devicePath, isPrefixMatch); + } else { + return getMeasurementPathsWithPointQuery(devicePath, isPrefixMatch); + } + } + + private List getMeasurementPathsWithPointQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List res = new LinkedList<>(); + String path = devicePath.getFullPath(); + Map tags = pathToTags(path); + path = tagsToPath(tags); + DeviceEntry deviceEntry = idTable.getDeviceEntry(path); + if (deviceEntry == null) return res; + Map schemaMap = deviceEntry.getMeasurementMap(); + for (String measurement : schemaMap.keySet()) { + SchemaEntry schemaEntry = schemaMap.get(measurement); + MeasurementPath measurementPath = + new MeasurementPath( + path, + measurement, + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); + res.add(measurementPath); + } + + return res; + } + + private List getMeasurementPathsWithBatchQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List res = new LinkedList<>(); + List deviceIDs = getDeviceIdFromInvertedIndex(devicePath); + for (IDeviceID deviceID : deviceIDs) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map schemaMap = deviceEntry.getMeasurementMap(); + if (deviceID instanceof SHA256DeviceID) { + for (String measurement : schemaMap.keySet()) { + SchemaEntry schemaEntry = schemaMap.get(measurement); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + MeasurementPath measurementPath = + new MeasurementPath( + new PartialPath(diskSchemaEntry.seriesKey), + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); + res.add(measurementPath); + } + } else { + for (String measurement : schemaMap.keySet()) { + SchemaEntry schemaEntry = schemaMap.get(measurement); + MeasurementPath measurementPath = + new MeasurementPath( + deviceID.toStringID(), + measurement, + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(deviceEntry.isAligned()); + res.add(measurementPath); + } + } + } + return res; + } + + @Override + public Pair, Integer> getMeasurementPathsWithAlias( + PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) + throws MetadataException { + List res = getMeasurementPaths(pathPattern, isPrefixMatch); + Pair, Integer> result = new Pair<>(res, 0); + return result; + } + + @Override + public List fetchSchema( + PartialPath pathPattern, Map templateMap) throws MetadataException { + return null; + } + + @Override + public Pair, Integer> showTimeseries( + ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { + List res = new ArrayList<>(); + Pair, Integer> result = new Pair<>(res, 0); + String path = plan.getPath().getFullPath(); + if (!path.endsWith(TAIL)) { + Map tags = pathToTags(path); + path = tagsToPath(tags); + DeviceEntry deviceEntry = idTable.getDeviceEntry(path); + if (deviceEntry != null) { + Map measurementMap = deviceEntry.getMeasurementMap(); + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + res.add( + new ShowTimeSeriesResult( + path + "." + m, + "null", + storageGroupFullPath, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>())); + } + } + return result; + } + List deviceIDs = getDeviceIdFromInvertedIndex(plan.getPath()); + for (IDeviceID deviceID : deviceIDs) { + getTimeSeriesResultOfDeviceFromIDTable(res, deviceID); + } + return result; + } + + private List getDeviceIdFromInvertedIndex(PartialPath devicePath) + throws MetadataException { + String path = devicePath.getFullPath(); + if (path.endsWith(TAIL)) { + path = path.substring(0, path.length() - TAIL.length()); + devicePath = new PartialPath(path); + } + if (devicePath.getFullPath().length() <= storageGroupFullPath.length()) { + return deviceIDS; + } else { + List res = new LinkedList<>(); + List ids = getDeviceIDsByInvertedIndex(devicePath); + if (ids.size() > 0) { + for (int id : ids) { + res.add(deviceIDS.get(id)); + } + } + return res; + } + } + + private void getTimeSeriesResultOfDeviceFromIDTable( + List res, IDeviceID deviceID) { + Map measurementMap = + idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap(); + if (deviceID instanceof SHA256DeviceID) { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + res.add( + new ShowTimeSeriesResult( + diskSchemaEntry.seriesKey, + "null", + storageGroupFullPath, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>())); + } + } else { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + res.add( + new ShowTimeSeriesResult( + deviceID.toStringID() + "." + m, + "null", + storageGroupFullPath, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>())); + } + } + } + + private List getDeviceIDsByInvertedIndex(PartialPath path) { + Map tags = pathToTags(path.getFullPath()); + List idsCollection = new ArrayList<>(tags.keySet().size()); + for (String tagKey : tags.keySet()) { + if (!tagInvertedIndex.containsKey(tagKey) + || !tagInvertedIndex.get(tagKey).containsKey(tags.get(tagKey))) { + return new ArrayList<>(); + } + List ids = tagInvertedIndex.get(tagKey).get(tags.get(tagKey)); + idsCollection.add(new ArrayList(ids)); + } + if (idsCollection.size() == 0) return new ArrayList<>(); + List ids = idsCollection.get(0); + for (int i = 1; i < idsCollection.size(); i++) { + List list = idsCollection.get(i); + ids.retainAll(list); + } + return ids; + } + + @Override + public List getAllMeasurementByDevicePath(PartialPath devicePath) + throws PathNotExistException { + throw new UnsupportedOperationException(""); + } + + @Override + public IMNode getDeviceNode(PartialPath path) throws MetadataException { + DeviceEntry deviceEntry = idTable.getDeviceEntry(path.getFullPath()); + if (deviceEntry == null) throw new PathNotExistException(path.getFullPath()); + return new EntityMNode(storageGroupMNode, path.getFullPath()); + } + + @Override + public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void upsertTagsAndAttributes( + String alias, + Map tagsMap, + Map attributesMap, + PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void addAttributes(Map attributesMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void addTags(Map tagsMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) + throws MetadataException, IOException { + PartialPath devicePath = plan.getDevicePath(); + Map tags = pathToTags(devicePath.getFullPath()); + devicePath = new PartialPath(tagsToPath(tags)); + plan.setDevicePath(devicePath); + String[] measurementList = plan.getMeasurements(); + IMeasurementMNode[] measurementMNodes = plan.getMeasurementMNodes(); + checkAlignedAndAutoCreateSeries(plan); + IMNode deviceMNode = getDeviceNode(devicePath); + IMeasurementMNode measurementMNode; + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + Map schemaMap = deviceEntry.getMeasurementMap(); + for (int i = 0; i < measurementList.length; i++) { + SchemaEntry schemaEntry = schemaMap.get(measurementList[i]); + measurementMNode = new InsertMeasurementMNode(measurementList[i], schemaEntry, null); + // check type is match + try { + SchemaRegionUtils.checkDataTypeMatch(plan, i, schemaEntry.getTSDataType()); + } catch (DataTypeMismatchException mismatchException) { + if (!config.isEnablePartialInsert()) { + throw mismatchException; + } else { + // mark failed measurement + plan.markFailedMeasurementInsertion(i, mismatchException); + continue; + } + } + measurementMNodes[i] = measurementMNode; + } + plan.setDeviceID(deviceEntry.getDeviceID()); + plan.setDevicePath(new PartialPath(deviceEntry.getDeviceID().toStringID(), false)); + return deviceMNode; + } + + @Override + public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( + PartialPath devicePath, + String[] measurements, + Function getDataType, + TSEncoding[] encodings, + CompressionType[] compressionTypes, + boolean aligned) + throws MetadataException { + List measurementSchemaInfoList = new ArrayList<>(measurements.length); + for (int i = 0; i < measurements.length; i++) { + SchemaEntry schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + if (schemaEntry == null) { + if (config.isAutoCreateSchemaEnabled()) { + if (aligned) { + TSDataType dataType = getDataType.apply(i); + internalAlignedCreateTimeseries( + devicePath, + Collections.singletonList(measurements[i]), + Collections.singletonList(dataType), + Collections.singletonList( + encodings[i] == null ? getDefaultEncoding(dataType) : encodings[i]), + Collections.singletonList( + compressionTypes[i] == null + ? TSFileDescriptor.getInstance().getConfig().getCompressor() + : compressionTypes[i])); + + } else { + internalCreateTimeseries( + devicePath.concatNode(measurements[i]), + getDataType.apply(i), + encodings[i], + compressionTypes[i]); + } + } + schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + } + measurementSchemaInfoList.add( + new MeasurementSchemaInfo( + measurements[i], + new MeasurementSchema( + measurements[i], + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType()), + null)); + } + return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList); + } + + private SchemaEntry getSchemaEntry(String devicePath, String measurementName) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry == null) return null; + return deviceEntry.getSchemaEntry(measurementName); + } + + private DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( + PartialPath devicePath, + String[] measurements, + Function getDataType, + boolean aligned) + throws MetadataException { + List measurementSchemaInfoList = new ArrayList<>(measurements.length); + for (int i = 0; i < measurements.length; i++) { + SchemaEntry schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + if (schemaEntry == null) { + if (config.isAutoCreateSchemaEnabled()) { + if (aligned) { + internalAlignedCreateTimeseries( + devicePath, + Collections.singletonList(measurements[i]), + Collections.singletonList(getDataType.apply(i))); + + } else { + internalCreateTimeseries(devicePath.concatNode(measurements[i]), getDataType.apply(i)); + } + } + schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + } + measurementSchemaInfoList.add( + new MeasurementSchemaInfo( + measurements[i], + new MeasurementSchema( + measurements[i], + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType()), + null)); + } + return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList); + } + + private void checkAlignedAndAutoCreateSeries(InsertPlan plan) throws MetadataException { + String[] measurementList = plan.getMeasurements(); + try { + if (plan.isAligned()) { + internalAlignedCreateTimeseries( + plan.getDevicePath(), + Arrays.asList(measurementList), + Arrays.asList(plan.getDataTypes())); + } else { + internalCreateTimeseries( + plan.getDevicePath().concatNode(measurementList[0]), plan.getDataTypes()[0]); + } + } catch (MetadataException e) { + if (!(e instanceof PathAlreadyExistException)) { + throw e; + } + } + } + + private void internalCreateTimeseries(PartialPath path, TSDataType dataType) + throws MetadataException { + createTimeseries( + path, + dataType, + getDefaultEncoding(dataType), + TSFileDescriptor.getInstance().getConfig().getCompressor(), + Collections.emptyMap()); + } + + /** create timeseries ignoring PathAlreadyExistException */ + private void internalCreateTimeseries( + PartialPath path, TSDataType dataType, TSEncoding encoding, CompressionType compressor) + throws MetadataException { + if (encoding == null) { + encoding = getDefaultEncoding(dataType); + } + if (compressor == null) { + compressor = TSFileDescriptor.getInstance().getConfig().getCompressor(); + } + createTimeseries(path, dataType, encoding, compressor, Collections.emptyMap()); + } + + private void internalAlignedCreateTimeseries( + PartialPath prefixPath, List measurements, List dataTypes) + throws MetadataException { + List encodings = new ArrayList<>(); + List compressors = new ArrayList<>(); + for (TSDataType dataType : dataTypes) { + encodings.add(getDefaultEncoding(dataType)); + compressors.add(TSFileDescriptor.getInstance().getConfig().getCompressor()); + } + createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); + } + + /** create aligned timeseries ignoring PathAlreadyExistException */ + private void internalAlignedCreateTimeseries( + PartialPath prefixPath, + List measurements, + List dataTypes, + List encodings, + List compressors) + throws MetadataException { + createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); + } + + @Override + public Set getPathsSetTemplate(String templateName) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public Set getPathsUsingTemplate(String templateName) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public boolean isTemplateAppendable(Template template, List measurements) + throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template) + throws MetadataException {} + + @Override + public List getPathsUsingTemplate(int templateId) throws MetadataException { + return null; + } + + @Override + public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { + throw new UnsupportedOperationException(""); + } + + @Override + public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException { + throw new UnsupportedOperationException(""); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java new file mode 100644 index 000000000000..a47b66cfa0b9 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaConfig.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +/** tag schema region config */ +public class TagSchemaConfig { + + // the maximum number of device ids managed by a working memTable + private int numOfDeviceIdsInMemTable = 65536; + + // the size of wal buffer used to store a wal record + private int walBufferSize = 1024 * 1024; + + public int getNumOfDeviceIdsInMemTable() { + return numOfDeviceIdsInMemTable; + } + + public void setNumOfDeviceIdsInMemTable(int numOfDeviceIdsInMemTable) { + this.numOfDeviceIdsInMemTable = numOfDeviceIdsInMemTable; + } + + public int getWalBufferSize() { + return walBufferSize; + } + + public void setWalBufferSize(int walBufferSize) { + this.walBufferSize = walBufferSize; + } + + @Override + public String toString() { + return "TagSchemaConfig[" + + "numOfDeviceIdsInMemTable=" + + numOfDeviceIdsInMemTable + + ", walBufferSize=" + + walBufferSize + + "]"; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java new file mode 100644 index 000000000000..881a0e28a722 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaDescriptor.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +import org.apache.iotdb.commons.conf.IoTDBConstant; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +/** manager tag schema config */ +public class TagSchemaDescriptor { + private static final Logger logger = LoggerFactory.getLogger(TagSchemaDescriptor.class); + + private static final String TAG_SCHEMA_CONFIG_FILE_NAME = "schema-tag.properties"; + + private final TagSchemaConfig conf = new TagSchemaConfig(); + + private TagSchemaDescriptor() { + loadProperties(); + } + + public static TagSchemaDescriptor getInstance() { + return TagSchemaDescriptorHolder.INSTANCE; + } + + private void loadProperties() { + String iotDBHomePath = System.getProperty(IoTDBConstant.IOTDB_HOME, null); + if (iotDBHomePath == null) { + logger.warn( + "Cannot find IOTDB_HOME environment variable when loading " + + "config file {}, use default configuration", + TAG_SCHEMA_CONFIG_FILE_NAME); + return; + } + String tagSchemaConfigPath = + iotDBHomePath + + File.separatorChar + + "conf" + + File.separatorChar + + TAG_SCHEMA_CONFIG_FILE_NAME; + try (InputStream in = new BufferedInputStream(new FileInputStream(tagSchemaConfigPath))) { + Properties properties = new Properties(); + properties.load(in); + conf.setWalBufferSize( + Integer.parseInt( + properties.getProperty("wal_buffer_size", String.valueOf(conf.getWalBufferSize())))); + conf.setNumOfDeviceIdsInMemTable( + Integer.parseInt( + properties.getProperty( + "num_of_deviceIds_in_memTable", + String.valueOf(conf.getNumOfDeviceIdsInMemTable())))); + } catch (FileNotFoundException e) { + logger.warn("Fail to find tag schema region config file {}", tagSchemaConfigPath); + } catch (IOException e) { + logger.warn("Cannot load tag schema region config file, use default configuration"); + } + } + + public TagSchemaConfig getTagSchemaConfig() { + return conf; + } + + private static class TagSchemaDescriptorHolder { + + private static final TagSchemaDescriptor INSTANCE = new TagSchemaDescriptor(); + + private TagSchemaDescriptorHolder() {} + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java new file mode 100644 index 000000000000..93ee6e154029 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java @@ -0,0 +1,959 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +import org.apache.iotdb.common.rpc.thrift.TSchemaNode; +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.file.SystemFileFactory; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.AlignedTimeseriesException; +import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; +import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.exception.metadata.SchemaDirCreationFailureException; +import org.apache.iotdb.db.metadata.LocalSchemaProcessor; +import org.apache.iotdb.db.metadata.idtable.IDTable; +import org.apache.iotdb.db.metadata.idtable.IDTableManager; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.InsertMeasurementMNode; +import org.apache.iotdb.db.metadata.idtable.entry.SHA256DeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.metadata.mnode.EntityMNode; +import org.apache.iotdb.db.metadata.mnode.IMNode; +import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; +import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; +import org.apache.iotdb.db.metadata.path.MeasurementPath; +import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.metadata.schemaregion.SchemaRegionUtils; +import org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist.DeviceIDList; +import org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist.IDeviceIDList; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.TagInvertedIndex; +import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.MeasurementPathUtils; +import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.PathTagConverterUtils; +import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.ShowTimeSeriesResultUtils; +import org.apache.iotdb.db.metadata.template.Template; +import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.mpp.common.schematree.MeasurementSchemaInfo; +import org.apache.iotdb.db.qp.physical.crud.InsertPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplateInClusterPlan; +import org.apache.iotdb.db.qp.physical.sys.ActivateTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.AutoCreateDeviceMNodePlan; +import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan; +import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan; +import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan; +import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.dataset.ShowDevicesResult; +import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; +import org.apache.iotdb.external.api.ISeriesNumerLimiter; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; + +/** tag schema region */ +public class TagSchemaRegion implements ISchemaRegion { + private static final Logger logger = LoggerFactory.getLogger(TagSchemaRegion.class); + + protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + // when a path ends with ".**", it represents batch processing + private final String TAIL = ".**"; + + private final IStorageGroupMNode storageGroupMNode; + private final String storageGroupFullPath; + private final SchemaRegionId schemaRegionId; + private final String schemaRegionDirPath; + + // tag inverted index + private final TagInvertedIndex tagInvertedIndex; + + // manager device id -> INT32 id + private final IDeviceIDList deviceIDList; + + // manager timeSeries + private final IDTable idTable; + + private final ISeriesNumerLimiter seriesNumerLimiter; + + public TagSchemaRegion( + PartialPath storageGroup, + SchemaRegionId schemaRegionId, + IStorageGroupMNode storageGroupMNode, + ISeriesNumerLimiter seriesNumerLimiter) + throws MetadataException { + storageGroupFullPath = storageGroup.getFullPath(); + this.schemaRegionId = schemaRegionId; + String storageGroupDirPath = config.getSchemaDir() + File.separator + storageGroupFullPath; + schemaRegionDirPath = storageGroupDirPath + File.separator + schemaRegionId.getId(); + this.storageGroupMNode = storageGroupMNode; + this.seriesNumerLimiter = seriesNumerLimiter; + idTable = IDTableManager.getInstance().getIDTable(storageGroup); + tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath); + deviceIDList = new DeviceIDList(schemaRegionDirPath); + init(); + } + + @Override + public void init() throws MetadataException { + // must enableIDTableLogFile or deviceIDTransformationMethod=="Plain" + if (!config.isEnableIDTableLogFile() + && config.getDeviceIDTransformationMethod().equals("SHA256")) { + throw new MetadataException( + "enableIDTableLogFile OR deviceIDTransformationMethod==\"Plain\""); + } + File schemaRegionFolder = SystemFileFactory.INSTANCE.getFile(schemaRegionDirPath); + if (!schemaRegionFolder.exists()) { + if (schemaRegionFolder.mkdirs()) { + logger.info("create schema region folder {}", schemaRegionDirPath); + } else { + if (!schemaRegionFolder.exists()) { + logger.error("create schema region folder {} failed.", schemaRegionDirPath); + throw new SchemaDirCreationFailureException(schemaRegionDirPath); + } + } + } + logger.info("initialized successfully: {}", this); + } + + @Override + @TestOnly + public void clear() { + try { + tagInvertedIndex.clear(); + deviceIDList.clear(); + } catch (IOException e) { + logger.error("clear tag inverted index failed", e); + } + } + + @Override + public void forceMlog() { + // no need to record mlog + } + + @Override + public SchemaRegionId getSchemaRegionId() { + return schemaRegionId; + } + + @Override + public String getStorageGroupFullPath() { + return storageGroupFullPath; + } + + @Override + public void deleteSchemaRegion() throws MetadataException { + clear(); + SchemaRegionUtils.deleteSchemaRegionFolder(schemaRegionDirPath, logger); + } + + @Override + public boolean createSnapshot(File snapshotDir) { + // todo implement this + throw new UnsupportedOperationException("Tag mode currently doesn't support snapshot feature."); + } + + @Override + public void loadSnapshot(File latestSnapshotRootDir) { + // todo implement this + throw new UnsupportedOperationException("Tag mode currently doesn't support snapshot feature."); + } + + private void createTagInvertedIndex(PartialPath devicePath) { + IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath); + Map tagsMap = + PathTagConverterUtils.pathToTags(storageGroupFullPath, devicePath.getFullPath()); + synchronized (deviceIDList) { + deviceIDList.add(deviceID); + tagInvertedIndex.addTags(tagsMap, deviceIDList.size() - 1); + } + } + + private List getDeviceIDsByInvertedIndex(PartialPath path) { + Map tags = + PathTagConverterUtils.pathToTags(storageGroupFullPath, path.getFullPath()); + return tagInvertedIndex.getMatchedIDs(tags); + } + + private void createTimeseries( + PartialPath path, + TSDataType dataType, + TSEncoding encoding, + CompressionType compressor, + Map props) + throws MetadataException { + createTimeseries( + new CreateTimeSeriesPlan(path, dataType, encoding, compressor, props, null, null, null), 0); + } + + private void createAlignedTimeSeries( + PartialPath prefixPath, + List measurements, + List dataTypes, + List encodings, + List compressors) + throws MetadataException { + createAlignedTimeSeries( + new CreateAlignedTimeSeriesPlan( + prefixPath, measurements, dataTypes, encodings, compressors, null, null, null)); + } + + @Override + public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws MetadataException { + PartialPath devicePath = plan.getPath().getDevicePath(); + PartialPath path = + new PartialPath( + PathTagConverterUtils.pathToTagsSortPath(storageGroupFullPath, devicePath.getFullPath()) + + "." + + plan.getPath().getMeasurement()); + plan.setPath(path); + devicePath = plan.getPath().getDevicePath(); + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is not aligned, please use createTimeseries or change entity.", + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } else if (deviceEntry.getMeasurementMap().containsKey(plan.getPath().getMeasurement())) { + throw new PathAlreadyExistException( + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } + } + idTable.createTimeseries(plan); + // write the device path for the first time + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + @Override + public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws MetadataException { + PartialPath devicePath = plan.getPrefixPath(); + PartialPath path = + new PartialPath( + PathTagConverterUtils.pathToTagsSortPath( + storageGroupFullPath, devicePath.getFullPath())); + plan.setPrefixPath(path); + devicePath = plan.getPrefixPath(); + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (!deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is aligned, please use createAlignedTimeseries or change entity.", + devicePath.getFullPath()); + } else { + filterExistingMeasurements(plan, deviceEntry.getMeasurementMap().keySet()); + if (plan.getMeasurements().size() == 0) + throw new PathAlreadyExistException(devicePath.getFullPath()); + } + } + idTable.createAlignedTimeseries(plan); + // write the device path for the first time + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + private void filterExistingMeasurements( + CreateAlignedTimeSeriesPlan plan, Set measurementSet) { + List measurements = plan.getMeasurements(); + List dataTypes = plan.getDataTypes(); + List encodings = plan.getEncodings(); + List compressors = plan.getCompressors(); + + List tmpMeasurements = new LinkedList<>(); + List tmpDataTypes = new LinkedList<>(); + List tmpEncodings = new LinkedList<>(); + List tmpCompressors = new LinkedList<>(); + for (int i = 0; i < measurements.size(); i++) { + String measurement = measurements.get(i); + if (!measurementSet.contains(measurement)) { + tmpMeasurements.add(measurements.get(i)); + tmpDataTypes.add(dataTypes.get(i)); + tmpEncodings.add(encodings.get(i)); + tmpCompressors.add(compressors.get(i)); + } + } + plan.setMeasurements(tmpMeasurements); + plan.setDataTypes(tmpDataTypes); + plan.setEncodings(tmpEncodings); + plan.setCompressors(tmpCompressors); + } + + @Override + public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + throw new UnsupportedOperationException("deleteTimeseries"); + } + + @Override + public int constructSchemaBlackList(PathPatternTree patternTree) throws MetadataException { + throw new UnsupportedOperationException("constructSchemaBlackList"); + } + + @Override + public void rollbackSchemaBlackList(PathPatternTree patternTree) throws MetadataException { + throw new UnsupportedOperationException("rollbackSchemaBlackList"); + } + + @Override + public List fetchSchemaBlackList(PathPatternTree patternTree) + throws MetadataException { + throw new UnsupportedOperationException("fetchSchemaBlackList"); + } + + @Override + public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException { + throw new UnsupportedOperationException("deleteTimeseriesInBlackList"); + } + + @Override + public void autoCreateDeviceMNode(AutoCreateDeviceMNodePlan plan) throws MetadataException { + throw new UnsupportedOperationException("autoCreateDeviceMNode"); + } + + @Override + public boolean isPathExist(PartialPath path) throws MetadataException { + throw new UnsupportedOperationException("isPathExist"); + } + + @Override + public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + int res = 0; + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + for (IDeviceID deviceID : deviceIDs) { + res += idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap().keySet().size(); + } + return res; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) + throws MetadataException { + throw new UnsupportedOperationException("getAllTimeseriesCount"); + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) + throws MetadataException { + throw new UnsupportedOperationException("getAllTimeseriesCount"); + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { + throw new UnsupportedOperationException("getMeasurementCountGroupByLevel"); + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, + int level, + boolean isPrefixMatch, + String key, + String value, + boolean isContains) + throws MetadataException { + throw new UnsupportedOperationException("getMeasurementCountGroupByLevel"); + } + + @Override + public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + synchronized (deviceIDList) { + if (pathPattern.getFullPath().length() <= storageGroupFullPath.length()) { + return deviceIDList.size(); + } else { + return getDeviceIDsByInvertedIndex(pathPattern).size(); + } + } + } + + @Override + public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) + throws MetadataException { + throw new UnsupportedOperationException("getNodesCountInGivenLevel"); + } + + @Override + public List getNodesListInGivenLevel( + PartialPath pathPattern, + int nodeLevel, + boolean isPrefixMatch, + LocalSchemaProcessor.StorageGroupFilter filter) + throws MetadataException { + throw new UnsupportedOperationException("getNodesListInGivenLevel"); + } + + @Override + public Set getChildNodePathInNextLevel(PartialPath pathPattern) + throws MetadataException { + throw new UnsupportedOperationException("getChildNodePathInNextLevel"); + } + + @Override + public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { + throw new UnsupportedOperationException("getChildNodeNameInNextLevel"); + } + + @Override + public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { + throw new UnsupportedOperationException("getBelongedDevices"); + } + + @Override + public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + Set matchedDevices = new HashSet<>(); + String devicePath = pathPattern.getFullPath(); + // exact query + if (!devicePath.endsWith(TAIL) && !devicePath.equals(storageGroupFullPath)) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry != null) { + matchedDevices.add(pathPattern); + } + return matchedDevices; + } + List devicePaths = getDevicePaths(deviceIDs); + for (String path : devicePaths) { + matchedDevices.add(new PartialPath(path)); + } + return matchedDevices; + } + + private List getDevicePaths(List deviceIDS) { + List devicePaths = new ArrayList<>(); + if (config.getDeviceIDTransformationMethod().equals("SHA256")) { + List schemaEntries = new ArrayList<>(); + for (IDeviceID deviceID : deviceIDS) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map map = deviceEntry.getMeasurementMap(); + // For each device, only one SchemaEntry needs to be obtained + for (Map.Entry entry : map.entrySet()) { + schemaEntries.add(entry.getValue()); + break; + } + } + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + for (DiskSchemaEntry diskSchemaEntry : diskSchemaEntries) { + devicePaths.add(diskSchemaEntry.getDevicePath()); + } + } else { + for (IDeviceID deviceID : deviceIDS) { + devicePaths.add(deviceID.toStringID()); + } + } + return devicePaths; + } + + private List getSchemaEntries(List deviceIDS) { + List schemaEntries = new ArrayList<>(); + for (IDeviceID deviceID : deviceIDS) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map schemaMap = deviceEntry.getMeasurementMap(); + for (Map.Entry entry : schemaMap.entrySet()) { + schemaEntries.add(entry.getValue()); + } + } + return schemaEntries; + } + + private List getMeasurementPaths(List deviceIDS) + throws IllegalPathException { + List measurementPaths = new ArrayList<>(); + if (config.getDeviceIDTransformationMethod().equals("SHA256")) { + List schemaEntries = getSchemaEntries(deviceIDS); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + for (DiskSchemaEntry diskSchemaEntry : diskSchemaEntries) { + MeasurementPath measurementPath = + MeasurementPathUtils.generateMeasurementPath(diskSchemaEntry); + measurementPaths.add(measurementPath); + } + } else { + for (IDeviceID deviceID : deviceIDS) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(deviceID.toStringID()); + Map schemaMap = deviceEntry.getMeasurementMap(); + for (Map.Entry entry : schemaMap.entrySet()) { + MeasurementPath measurementPath = + MeasurementPathUtils.generateMeasurementPath( + deviceID.toStringID(), entry.getKey(), entry.getValue(), deviceEntry.isAligned()); + measurementPaths.add(measurementPath); + } + } + } + return measurementPaths; + } + + @Override + public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) + throws MetadataException { + throw new UnsupportedOperationException("getMatchedDevices"); + } + + @Override + public List getMeasurementPaths(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + PartialPath devicePath = pathPattern.getDevicePath(); + if (devicePath.getFullPath().endsWith(TAIL)) { + return getMeasurementPathsWithBatchQuery(devicePath, isPrefixMatch); + } else { + return getMeasurementPathsWithPointQuery(devicePath, isPrefixMatch); + } + } + + private List getMeasurementPathsWithPointQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List measurementPaths = new LinkedList<>(); + String path = + PathTagConverterUtils.pathToTagsSortPath(storageGroupFullPath, devicePath.getFullPath()); + DeviceEntry deviceEntry = idTable.getDeviceEntry(path); + if (deviceEntry == null) return measurementPaths; + Map schemaMap = deviceEntry.getMeasurementMap(); + for (Map.Entry entry : schemaMap.entrySet()) { + MeasurementPath measurementPath = + MeasurementPathUtils.generateMeasurementPath( + path, entry.getKey(), entry.getValue(), deviceEntry.isAligned()); + measurementPaths.add(measurementPath); + } + return measurementPaths; + } + + private List getMeasurementPathsWithBatchQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List deviceIDs = getDeviceIdFromInvertedIndex(devicePath); + return getMeasurementPaths(deviceIDs); + } + + @Override + public Pair, Integer> getMeasurementPathsWithAlias( + PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch) + throws MetadataException { + List res = getMeasurementPaths(pathPattern, isPrefixMatch); + Pair, Integer> result = new Pair<>(res, 0); + return result; + } + + @Override + public List fetchSchema( + PartialPath pathPattern, Map templateMap) throws MetadataException { + throw new UnsupportedOperationException("fetchSchema"); + } + + @Override + public Pair, Integer> showTimeseries( + ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { + List ShowTimeSeriesResults = new ArrayList<>(); + Pair, Integer> result = new Pair<>(ShowTimeSeriesResults, 0); + String path = plan.getPath().getFullPath(); + // point query + if (!path.endsWith(TAIL)) { + path = PathTagConverterUtils.pathToTagsSortPath(storageGroupFullPath, path); + DeviceEntry deviceEntry = idTable.getDeviceEntry(path); + if (deviceEntry != null) { + Map measurementMap = deviceEntry.getMeasurementMap(); + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + ShowTimeSeriesResults.add( + ShowTimeSeriesResultUtils.generateShowTimeSeriesResult( + storageGroupFullPath, path, m, schemaEntry)); + } + } + return result; + } + // batch query + List deviceIDs = getDeviceIdFromInvertedIndex(plan.getPath()); + for (IDeviceID deviceID : deviceIDs) { + getTimeSeriesResultOfDeviceFromIDTable(ShowTimeSeriesResults, deviceID); + } + return result; + } + + private void getTimeSeriesResultOfDeviceFromIDTable( + List ShowTimeSeriesResults, IDeviceID deviceID) { + Map measurementMap = + idTable.getDeviceEntry(deviceID.toStringID()).getMeasurementMap(); + if (deviceID instanceof SHA256DeviceID) { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = idTable.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + ShowTimeSeriesResults.add( + ShowTimeSeriesResultUtils.generateShowTimeSeriesResult( + storageGroupFullPath, diskSchemaEntry.seriesKey, schemaEntry)); + } + } else { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + ShowTimeSeriesResults.add( + ShowTimeSeriesResultUtils.generateShowTimeSeriesResult( + storageGroupFullPath, deviceID.toStringID(), m, schemaEntry)); + } + } + } + + private List getDeviceIdFromInvertedIndex(PartialPath devicePath) + throws MetadataException { + String path = devicePath.getFullPath(); + if (path.endsWith(TAIL)) { + path = path.substring(0, path.length() - TAIL.length()); + devicePath = new PartialPath(path); + } + synchronized (deviceIDList) { + if (devicePath.getFullPath().length() <= storageGroupFullPath.length()) { + return deviceIDList.getAllDeviceIDS(); + } else { + List IDS = new LinkedList<>(); + List ids = getDeviceIDsByInvertedIndex(devicePath); + if (ids.size() > 0) { + for (int id : ids) { + IDS.add(deviceIDList.get(id)); + } + } + return IDS; + } + } + } + + @Override + public List getAllMeasurementByDevicePath(PartialPath devicePath) + throws PathNotExistException { + throw new UnsupportedOperationException("getAllMeasurementByDevicePath"); + } + + @Override + public IMNode getDeviceNode(PartialPath path) throws MetadataException { + DeviceEntry deviceEntry = idTable.getDeviceEntry(path.getFullPath()); + if (deviceEntry == null) throw new PathNotExistException(path.getFullPath()); + return new EntityMNode(storageGroupMNode, path.getFullPath()); + } + + @Override + public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { + throw new UnsupportedOperationException("getMeasurementMNode"); + } + + @Override + public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException { + throw new UnsupportedOperationException("changeAlias"); + } + + @Override + public void upsertTagsAndAttributes( + String alias, + Map tagsMap, + Map attributesMap, + PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException("upsertTagsAndAttributes"); + } + + @Override + public void addAttributes(Map attributesMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException("addAttributes"); + } + + @Override + public void addTags(Map tagsMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException("addTags"); + } + + @Override + public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException("dropTagsOrAttributes"); + } + + @Override + public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException("setTagsOrAttributesValue"); + } + + @Override + public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException("renameTagOrAttributeKey"); + } + + @Override + public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) + throws MetadataException, IOException { + PartialPath devicePath = plan.getDevicePath(); + devicePath = + new PartialPath( + PathTagConverterUtils.pathToTagsSortPath( + storageGroupFullPath, devicePath.getFullPath())); + plan.setDevicePath(devicePath); + String[] measurementList = plan.getMeasurements(); + IMeasurementMNode[] measurementMNodes = plan.getMeasurementMNodes(); + checkAlignedAndAutoCreateSeries(plan); + IMNode deviceMNode = getDeviceNode(devicePath); + IMeasurementMNode measurementMNode; + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath.getFullPath()); + Map schemaMap = deviceEntry.getMeasurementMap(); + for (int i = 0; i < measurementList.length; i++) { + SchemaEntry schemaEntry = schemaMap.get(measurementList[i]); + measurementMNode = new InsertMeasurementMNode(measurementList[i], schemaEntry, null); + // check type is match + try { + SchemaRegionUtils.checkDataTypeMatch(plan, i, schemaEntry.getTSDataType()); + } catch (DataTypeMismatchException mismatchException) { + if (!config.isEnablePartialInsert()) { + throw mismatchException; + } else { + // mark failed measurement + plan.markFailedMeasurementInsertion(i, mismatchException); + continue; + } + } + measurementMNodes[i] = measurementMNode; + } + plan.setDeviceID(deviceEntry.getDeviceID()); + plan.setDevicePath(new PartialPath(deviceEntry.getDeviceID().toStringID(), false)); + return deviceMNode; + } + + @Override + public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( + PartialPath devicePath, + String[] measurements, + Function getDataType, + TSEncoding[] encodings, + CompressionType[] compressionTypes, + boolean aligned) + throws MetadataException { + List measurementSchemaInfoList = new ArrayList<>(measurements.length); + for (int i = 0; i < measurements.length; i++) { + SchemaEntry schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + if (schemaEntry == null) { + if (config.isAutoCreateSchemaEnabled()) { + if (aligned) { + TSDataType dataType = getDataType.apply(i); + internalAlignedCreateTimeseries( + devicePath, + Collections.singletonList(measurements[i]), + Collections.singletonList(dataType), + Collections.singletonList( + encodings[i] == null ? getDefaultEncoding(dataType) : encodings[i]), + Collections.singletonList( + compressionTypes[i] == null + ? TSFileDescriptor.getInstance().getConfig().getCompressor() + : compressionTypes[i])); + + } else { + internalCreateTimeseries( + devicePath.concatNode(measurements[i]), + getDataType.apply(i), + encodings[i], + compressionTypes[i]); + } + } + schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + } + measurementSchemaInfoList.add( + new MeasurementSchemaInfo( + measurements[i], + new MeasurementSchema( + measurements[i], + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType()), + null)); + } + return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList); + } + + private SchemaEntry getSchemaEntry(String devicePath, String measurementName) { + DeviceEntry deviceEntry = idTable.getDeviceEntry(devicePath); + if (deviceEntry == null) return null; + return deviceEntry.getSchemaEntry(measurementName); + } + + private void checkAlignedAndAutoCreateSeries(InsertPlan plan) throws MetadataException { + String[] measurementList = plan.getMeasurements(); + try { + if (plan.isAligned()) { + internalAlignedCreateTimeseries( + plan.getDevicePath(), + Arrays.asList(measurementList), + Arrays.asList(plan.getDataTypes())); + } else { + internalCreateTimeseries( + plan.getDevicePath().concatNode(measurementList[0]), plan.getDataTypes()[0]); + } + } catch (MetadataException e) { + if (!(e instanceof PathAlreadyExistException)) { + throw e; + } + } + } + + /** create timeseries ignoring PathAlreadyExistException */ + private void internalCreateTimeseries(PartialPath path, TSDataType dataType) + throws MetadataException { + createTimeseries( + path, + dataType, + getDefaultEncoding(dataType), + TSFileDescriptor.getInstance().getConfig().getCompressor(), + Collections.emptyMap()); + } + + /** create timeseries ignoring PathAlreadyExistException */ + private void internalCreateTimeseries( + PartialPath path, TSDataType dataType, TSEncoding encoding, CompressionType compressor) + throws MetadataException { + if (encoding == null) { + encoding = getDefaultEncoding(dataType); + } + if (compressor == null) { + compressor = TSFileDescriptor.getInstance().getConfig().getCompressor(); + } + createTimeseries(path, dataType, encoding, compressor, Collections.emptyMap()); + } + + /** create aligned timeseries ignoring PathAlreadyExistException */ + private void internalAlignedCreateTimeseries( + PartialPath prefixPath, List measurements, List dataTypes) + throws MetadataException { + List encodings = new ArrayList<>(); + List compressors = new ArrayList<>(); + for (TSDataType dataType : dataTypes) { + encodings.add(getDefaultEncoding(dataType)); + compressors.add(TSFileDescriptor.getInstance().getConfig().getCompressor()); + } + createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); + } + + /** create aligned timeseries ignoring PathAlreadyExistException */ + private void internalAlignedCreateTimeseries( + PartialPath prefixPath, + List measurements, + List dataTypes, + List encodings, + List compressors) + throws MetadataException { + createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); + } + + @Override + public Set getPathsSetTemplate(String templateName) throws MetadataException { + throw new UnsupportedOperationException("getPathsSetTemplate"); + } + + @Override + public Set getPathsUsingTemplate(String templateName) throws MetadataException { + throw new UnsupportedOperationException("getPathsUsingTemplate"); + } + + @Override + public boolean isTemplateAppendable(Template template, List measurements) + throws MetadataException { + throw new UnsupportedOperationException("isTemplateAppendable"); + } + + @Override + public void setSchemaTemplate(SetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException("setSchemaTemplate"); + } + + @Override + public void unsetSchemaTemplate(UnsetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException("unsetSchemaTemplate"); + } + + @Override + public void setUsingSchemaTemplate(ActivateTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException("setUsingSchemaTemplate"); + } + + @Override + public void activateSchemaTemplate(ActivateTemplateInClusterPlan plan, Template template) + throws MetadataException { + throw new UnsupportedOperationException("activateSchemaTemplate"); + } + + @Override + public List getPathsUsingTemplate(int templateId) throws MetadataException { + throw new UnsupportedOperationException("getPathsUsingTemplate"); + } + + @Override + public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { + throw new UnsupportedOperationException("getMNodeForTrigger"); + } + + @Override + public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException { + throw new UnsupportedOperationException("releaseMNodeAfterDropTrigger"); + } + + @Override + public String toString() { + return "TagSchemaRegion{" + + "storageGroupFullPath='" + + storageGroupFullPath + + '\'' + + ", schemaRegionId=" + + schemaRegionId + + ", schemaRegionDirPath='" + + schemaRegionDirPath + + '\'' + + '}'; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java new file mode 100644 index 000000000000..6dda97a00aa1 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/AppendOnlyDeviceIDListFileManager.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist; + +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; + +/** for append-only writing of device id list to disk */ +public class AppendOnlyDeviceIDListFileManager { + + private static final Logger logger = + LoggerFactory.getLogger(AppendOnlyDeviceIDListFileManager.class); + + private static final String DEVICE_ID_LIST_FILE_NAME = "device_id.list"; + + private boolean isRecover; + + private String schemaDirPath; + + private File deviceIDSFile; + + private FileOutputStream outputStream; + + public AppendOnlyDeviceIDListFileManager(String schemaDirPath) { + this.schemaDirPath = schemaDirPath; + isRecover = true; + try { + initFile(); + outputStream = new FileOutputStream(deviceIDSFile, true); + } catch (IOException e) { + logger.error(e.getMessage()); + throw new IllegalArgumentException( + "can't initialize device id list file at " + deviceIDSFile); + } + } + + private void initFile() throws IOException { + File schemaDir = new File(schemaDirPath); + schemaDir.mkdirs(); + deviceIDSFile = new File(schemaDir, DEVICE_ID_LIST_FILE_NAME); + if (!deviceIDSFile.exists()) { + // create new file + deviceIDSFile.createNewFile(); + } + } + + /** + * write the device id to file + * + * @param deviceID device id + */ + public void write(String deviceID) { + try { + if (!isRecover) { + ReadWriteIOUtils.write(deviceID, outputStream); + } + } catch (IOException e) { + logger.error("failed to write device id: " + deviceID); + throw new IllegalArgumentException("can't write device id of " + deviceID); + } + } + + /** + * recover device id list + * + * @param deviceIDList device id list + */ + public void recover(DeviceIDList deviceIDList) { + logger.info("recover device id list using file {}", deviceIDSFile); + try (FileInputStream inputStream = new FileInputStream(deviceIDSFile)) { + while (inputStream.available() > 0) { + String deviceID = ReadWriteIOUtils.readString(inputStream); + deviceIDList.add(DeviceIDFactory.getInstance().getDeviceID(deviceID)); + } + } catch (IOException e) { + logger.info("device id list is incomplete, we will recover as much as we can."); + } + isRecover = false; + } + + @TestOnly + public void close() throws IOException { + try { + outputStream.close(); + outputStream = null; + } catch (IOException e) { + logger.error("close device id list file failed"); + throw e; + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java new file mode 100644 index 000000000000..4099248724d7 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDList.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist; + +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** manage device id -> int32 id */ +public class DeviceIDList implements IDeviceIDList { + + // use an array list to manage device id -> int id + private final List deviceIDS; + + private AppendOnlyDeviceIDListFileManager appendOnlyDeviceIDListFileManager; + + public DeviceIDList(String schemaDirPath) { + deviceIDS = new ArrayList<>(); + appendOnlyDeviceIDListFileManager = new AppendOnlyDeviceIDListFileManager(schemaDirPath); + recover(); + } + + public void recover() { + appendOnlyDeviceIDListFileManager.recover(this); + } + + @Override + public void add(IDeviceID deviceID) { + deviceIDS.add(deviceID); + appendOnlyDeviceIDListFileManager.write(deviceID.toStringID()); + } + + @Override + public IDeviceID get(int index) { + return deviceIDS.get(index); + } + + @Override + public int size() { + return deviceIDS.size(); + } + + @Override + public List getAllDeviceIDS() { + return new ArrayList<>(deviceIDS); + } + + @TestOnly + public void clear() throws IOException { + appendOnlyDeviceIDListFileManager.close(); + deviceIDS.clear(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java new file mode 100644 index 000000000000..3d461a4b7097 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/IDeviceIDList.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist; + +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; + +import java.io.IOException; +import java.util.List; + +/** manage device id -> int32 id */ +public interface IDeviceIDList { + + /** + * insert a device id + * + * @param deviceID device id + */ + void add(IDeviceID deviceID); + + /** + * get device id using int32 id + * + * @param index int32 id + * @return device id + */ + IDeviceID get(int index); + + /** + * returns the number of managed device ids + * + * @return the number of managed device ids + */ + int size(); + + /** + * get all managed device ids + * + * @return all managed device ids + */ + List getAllDeviceIDS(); + + @TestOnly + void clear() throws IOException; +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java new file mode 100644 index 000000000000..18c90ada1550 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +import org.apache.iotdb.lsm.context.DeleteRequestContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; + +import java.util.List; +import java.util.Map; + +/** tag inverted index interface */ +public interface ITagInvertedIndex { + + /** + * insert tags and id using insert request context + * + * @param context insert request context + */ + void addTags(InsertRequestContext context); + + /** + * insert tags and device id + * + * @param tags tags like: + * @param id INT32 device id + */ + void addTags(Map tags, int id); + + /** + * delete tags and id using delete request context + * + * @param context delete request context + */ + void removeTags(DeleteRequestContext context); + + /** + * delete tags and id using delete request context + * + * @param tags tags like: + * @param id INT32 device id + */ + void removeTags(Map tags, int id); + + /** + * get all matching device ids + * + * @param tags tags like: + * @return device ids + */ + List getMatchedIDs(Map tags); +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java new file mode 100644 index 000000000000..e161b5bf93b7 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaConfig; +import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion.DeletionManager; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion.InsertionManager; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query.QueryManager; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.recover.RecoverManager; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; +import org.apache.iotdb.lsm.context.DeleteRequestContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; +import org.apache.iotdb.lsm.context.QueryRequestContext; + +import org.roaringbitmap.RoaringBitmap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** tag reverse index implementation class */ +public class TagInvertedIndex implements ITagInvertedIndex { + + private static final Logger logger = LoggerFactory.getLogger(TagInvertedIndex.class); + + private static final TagSchemaConfig tagSchemaConfig = + TagSchemaDescriptor.getInstance().getTagSchemaConfig(); + + private InsertionManager insertionManager; + + private DeletionManager deletionManager; + + private QueryManager queryManager; + + private WALManager walManager; + + private RecoverManager recoverManager; + + // the maximum number of device ids managed by a working memTable + private int numOfDeviceIdsInMemTable; + + // (maxDeviceID / numOfDeviceIdsInMemTable) -> MemTable + private Map immutableMemTables; + + private MemTable workingMemTable; + + // the largest device id saved by the current MemTable + private int maxDeviceID; + + public TagInvertedIndex(String schemaDirPath) { + try { + walManager = new WALManager(schemaDirPath); + insertionManager = new InsertionManager(walManager); + deletionManager = new DeletionManager(walManager); + recoverManager = new RecoverManager(walManager); + queryManager = new QueryManager(); + workingMemTable = new MemTable(MemTable.WORKING); + immutableMemTables = new HashMap<>(); + numOfDeviceIdsInMemTable = tagSchemaConfig.getNumOfDeviceIdsInMemTable(); + maxDeviceID = 0; + recover(); + } catch (IOException e) { + logger.error("create TagInvertedIndex fail", e); + } + } + + public synchronized void recover() { + recoverManager.recover(this); + } + + /** + * insert tags and id using insert request context + * + * @param context insert request context + */ + @Override + public synchronized void addTags(InsertRequestContext context) { + int id = (int) context.getValue(); + // if the device id can not be saved to the current working MemTable + if (!inWorkingMemTable(id)) { + workingMemTable.setStatus(MemTable.IMMUTABLE); + immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); + workingMemTable = new MemTable(MemTable.WORKING); + } + MemTable memTable = workingMemTable; + maxDeviceID = id; + try { + insertionManager.process(memTable, context); + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + + /** + * insert tags and device id + * + * @param tags tags like: + * @param id INT32 device id + */ + @Override + public synchronized void addTags(Map tags, int id) { + // if the device id can not be saved to the current working MemTable + if (!inWorkingMemTable(id)) { + workingMemTable.setStatus(MemTable.IMMUTABLE); + immutableMemTables.put(maxDeviceID / numOfDeviceIdsInMemTable, workingMemTable); + workingMemTable = new MemTable(MemTable.WORKING); + } + MemTable memTable = workingMemTable; + maxDeviceID = id; + try { + for (Map.Entry tag : tags.entrySet()) { + addTag(memTable, tag.getKey(), tag.getValue(), id); + } + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + + /** + * delete tags and id using delete request context + * + * @param context delete request context + */ + @Override + public void removeTags(DeleteRequestContext context) { + int id = (int) context.getValue(); + MemTable memTable = null; + if (inWorkingMemTable(id)) { + memTable = workingMemTable; + } else { + memTable = immutableMemTables.get(id / numOfDeviceIdsInMemTable); + } + try { + deletionManager.process(memTable, context); + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + + /** + * delete tags and id using delete request context + * + * @param tags tags like: + * @param id INT32 device id + */ + @Override + public synchronized void removeTags(Map tags, int id) { + List memTables = new ArrayList<>(); + if (inWorkingMemTable(id)) { + memTables.add(workingMemTable); + } else { + memTables.add(immutableMemTables.get(id / numOfDeviceIdsInMemTable)); + } + try { + for (Map.Entry tag : tags.entrySet()) { + removeTag(memTables, tag.getKey(), tag.getValue(), id); + } + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + + /** + * get all matching device ids + * + * @param tags tags like: + * @return device ids + */ + @Override + public synchronized List getMatchedIDs(Map tags) { + List memTables = new ArrayList<>(); + memTables.add(workingMemTable); + memTables.addAll(immutableMemTables.values()); + RoaringBitmap roaringBitmap = new RoaringBitmap(); + int i = 0; + try { + for (Map.Entry tag : tags.entrySet()) { + RoaringBitmap rb = getMatchedIDs(memTables, tag.getKey(), tag.getValue()); + if (i == 0) roaringBitmap = rb; + else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb); + i++; + } + } catch (Exception e) { + logger.error(e.getMessage()); + } + return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList()); + } + + @Override + public String toString() { + return "TagInvertedIndex{" + + "numOfDeviceIdsInMemTable=" + + numOfDeviceIdsInMemTable + + ", workingMemTable=" + + workingMemTable + + ", immutableMemTables=" + + immutableMemTables + + ", maxDeviceID=" + + maxDeviceID + + '}'; + } + + /** + * determine whether the id can be saved to the current MemTable + * + * @param id INT32 device id + * @return return true if it can, otherwise return false + */ + private boolean inWorkingMemTable(int id) { + return id / numOfDeviceIdsInMemTable == maxDeviceID / numOfDeviceIdsInMemTable; + } + + private void addTag(MemTable memTable, String tagKey, String tagValue, int id) throws Exception { + InsertRequestContext insertContext = new InsertRequestContext(id, tagKey, tagValue); + insertionManager.process(memTable, insertContext); + } + + private void removeTag(List memTables, String tagKey, String tagValue, int id) + throws Exception { + DeleteRequestContext deleteContext = new DeleteRequestContext(id, tagKey, tagValue); + for (MemTable memTable : memTables) { + deletionManager.process(memTable, deleteContext); + } + } + + private RoaringBitmap getMatchedIDs(List memTables, String tagKey, String tagValue) + throws Exception { + QueryRequestContext queryContext = new QueryRequestContext(tagKey, tagValue); + for (MemTable memTable : memTables) { + queryManager.process(memTable, queryContext); + } + return (RoaringBitmap) queryContext.getResult(); + } + + @TestOnly + public void clear() throws IOException { + walManager.close(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java new file mode 100644 index 000000000000..51fd8ab031a2 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/DeletionManager.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; +import org.apache.iotdb.lsm.context.DeleteRequestContext; +import org.apache.iotdb.lsm.manager.BasicLsmManager; + +/** manage deletion to MemTable */ +public class DeletionManager extends BasicLsmManager { + + // use wal manager object to write wal file on deletion + private WALManager walManager; + + public DeletionManager(WALManager walManager) { + this.walManager = walManager; + initLevelProcess(); + } + + /** + * write wal file on deletion + * + * @param root root memory node + * @param context request context + * @throws Exception + */ + @Override + public void preProcess(MemTable root, DeleteRequestContext context) throws Exception { + if (!context.isRecover()) { + walManager.write(context); + } + } + + /** set the delete operation for each layer of memory nodes */ + private void initLevelProcess() { + this.nextLevel(new MemTableDeletion()) + .nextLevel(new MemChunkGroupDeletion()) + .nextLevel(new MemChunkDeletion()); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java new file mode 100644 index 000000000000..6b11f648710c --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.lsm.context.DeleteRequestContext; +import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; + +import java.util.List; + +/** deletion for MemChunk */ +public class MemChunkDeletion extends DeleteLevelProcess { + + /** + * MemChunk is the last layer of memory nodes, no children + * + * @param memNode memory node + * @param context request context + * @return null + */ + @Override + public List getChildren(MemChunk memNode, DeleteRequestContext context) { + return null; + } + + /** + * the delete method corresponding to the MemChunk node + * + * @param memNode memory node + * @param context deletion request context + */ + @Override + public void delete(MemChunk memNode, DeleteRequestContext context) { + Integer deviceID = (Integer) context.getValue(); + memNode.remove(deviceID); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java new file mode 100644 index 000000000000..a0f40a069aa9 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.lsm.context.DeleteRequestContext; +import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; + +import java.util.ArrayList; +import java.util.List; + +/** deletion for MemChunkGroup */ +public class MemChunkGroupDeletion extends DeleteLevelProcess { + + /** + * get all MemChunks that need to be processed in the current MemChunkGroup + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunks + */ + @Override + public List getChildren(MemChunkGroup memNode, DeleteRequestContext context) { + List memChunks = new ArrayList<>(); + String tagValue = (String) context.getKey(); + MemChunk child = memNode.get(tagValue); + if (child != null) memChunks.add(child); + return memChunks; + } + + /** + * the delete method corresponding to the MemChunkGroup node + * + * @param memNode memory node + * @param context deletion request context + */ + @Override + public void delete(MemChunkGroup memNode, DeleteRequestContext context) { + String tagValue = (String) context.getKey(); + MemChunk child = memNode.get(tagValue); + if (child == null || child.isEmpty()) { + memNode.remove(tagValue); + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java new file mode 100644 index 000000000000..4795585b32e5 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.DeleteRequestContext; +import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcess; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** deletion for MemTable */ +public class MemTableDeletion extends DeleteLevelProcess { + + /** + * get all MemChunkGroups that need to be processed in the current MemTable + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunkGroups + */ + @Override + public List getChildren(MemTable memNode, DeleteRequestContext context) { + if (memNode.isImmutable()) return new ArrayList<>(); + List memChunkGroups = new ArrayList<>(); + String tagKey = (String) context.getKey(); + MemChunkGroup child = memNode.get(tagKey); + if (child != null) memChunkGroups.add(child); + return memChunkGroups; + } + + /** + * the delete method corresponding to the MemTable node + * + * @param memNode memory node + * @param context deletion request context + */ + @Override + public void delete(MemTable memNode, DeleteRequestContext context) { + if (memNode.isImmutable()) { + Set deletionList = memNode.getDeletionList(); + if (!deletionList.contains(context.getValue())) { + deletionList.add((Integer) context.getValue()); + } + return; + } + String tagKey = (String) context.getKey(); + MemChunkGroup child = memNode.get(tagKey); + if (child == null || child.isEmpty()) { + memNode.remove(tagKey); + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java new file mode 100644 index 000000000000..583f8944fd14 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/InsertionManager.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; +import org.apache.iotdb.lsm.context.InsertRequestContext; +import org.apache.iotdb.lsm.manager.BasicLsmManager; + +import java.io.IOException; + +/** manage insertion to MemTable */ +public class InsertionManager extends BasicLsmManager { + + // use wal manager object to write wal file on insertion + private WALManager walManager; + + public InsertionManager(WALManager walManager) { + this.walManager = walManager; + initLevelProcess(); + } + + /** + * write wal file on insertion + * + * @param root root memory node + * @param context insert request context + * @throws Exception + */ + @Override + public void preProcess(MemTable root, InsertRequestContext context) throws IOException { + if (!context.isRecover()) { + walManager.write(context); + } + } + + /** set the insert operation for each layer of memory nodes */ + private void initLevelProcess() { + this.nextLevel(new MemTableInsertion()) + .nextLevel(new MemChunkGroupInsertion()) + .nextLevel(new MemChunkInsertion()); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java new file mode 100644 index 000000000000..5877b53f3d46 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.lsm.context.InsertRequestContext; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; + +import java.util.ArrayList; +import java.util.List; + +/** insertion for MemChunkGroup */ +public class MemChunkGroupInsertion extends InsertLevelProcess { + + /** + * get all MemChunks that need to be processed in the current MemChunkGroup + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunks + */ + @Override + public List getChildren(MemChunkGroup memNode, InsertRequestContext context) { + List memChunks = new ArrayList<>(); + String tagValue = (String) context.getKey(); + MemChunk child = memNode.get(tagValue); + if (child != null) memChunks.add(child); + return memChunks; + } + + /** + * the insert method corresponding to the MemChunkGroup node + * + * @param memNode memory node + * @param context insert request context + */ + @Override + public void insert(MemChunkGroup memNode, InsertRequestContext context) { + String tagValue = (String) context.getKey(); + memNode.put(tagValue); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java new file mode 100644 index 000000000000..1cf23bd2e1c0 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.lsm.context.InsertRequestContext; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; + +import java.util.List; + +/** insertion for MemChunk */ +public class MemChunkInsertion extends InsertLevelProcess { + + /** + * MemChunk is the last layer of memory nodes, no children + * + * @param memNode memory node + * @param context request context + * @return null + */ + @Override + public List getChildren(MemChunk memNode, InsertRequestContext context) { + return null; + } + + /** + * the insert method corresponding to the MemChunk node + * + * @param memNode memory node + * @param context insert request context + */ + @Override + public void insert(MemChunk memNode, InsertRequestContext context) { + Integer deviceID = (Integer) context.getValue(); + memNode.put(deviceID); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java new file mode 100644 index 000000000000..0d098e88999a --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.InsertRequestContext; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcess; + +import java.util.ArrayList; +import java.util.List; + +/** insertion for MemTable */ +public class MemTableInsertion extends InsertLevelProcess { + + /** + * get all MemChunkGroups that need to be processed in the current MemTable + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunkGroups + */ + @Override + public List getChildren(MemTable memNode, InsertRequestContext context) { + if (memNode.isImmutable()) return new ArrayList<>(); + List memChunkGroups = new ArrayList<>(); + String tagKey = (String) context.getKey(); + MemChunkGroup child = memNode.get(tagKey); + if (child != null) memChunkGroups.add(child); + return memChunkGroups; + } + + /** + * the insert method corresponding to the MemTable node + * + * @param memNode memory node + * @param context insert request context + */ + @Override + public void insert(MemTable memNode, InsertRequestContext context) { + if (memNode.isImmutable()) return; + String tagKey = (String) context.getKey(); + memNode.put(tagKey); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java new file mode 100644 index 000000000000..80610c1eaec6 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable; + +import org.roaringbitmap.RoaringBitmap; + +/** used to manage the device id collection */ +public class MemChunk { + + // manage the device id collection, see: https://github.com/RoaringBitmap/RoaringBitmap + private RoaringBitmap roaringBitmap; + + public MemChunk() { + roaringBitmap = new RoaringBitmap(); + } + + public boolean isEmpty() { + if (roaringBitmap == null) return true; + return roaringBitmap.isEmpty(); + } + + @Override + public String toString() { + return roaringBitmap.toString(); + } + + public void put(int id) { + roaringBitmap.add(id); + } + + public void remove(int id) { + roaringBitmap.remove(id); + } + + public RoaringBitmap getRoaringBitmap() { + return this.roaringBitmap; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java new file mode 100644 index 000000000000..c4dca031639a --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable; + +import java.util.HashMap; +import java.util.Map; + +/** used to manage tagValue -> MemChunk */ +public class MemChunkGroup { + + // manage tagValue -> MemChunk + private Map memChunkMap; + + public MemChunkGroup() { + memChunkMap = new HashMap<>(); + } + + public void put(String tagValue) { + if (!memChunkMap.containsKey(tagValue)) { + memChunkMap.put(tagValue, new MemChunk()); + } + } + + @Override + public String toString() { + return memChunkMap.toString(); + } + + public MemChunk get(String tagValue) { + return memChunkMap.get(tagValue); + } + + public void remove(String tagValue) { + memChunkMap.remove(tagValue); + } + + public boolean isEmpty() { + return memChunkMap.isEmpty(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java new file mode 100644 index 000000000000..5f9cd09c6284 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** used to manage tagKey -> MemChunkGroup */ +public class MemTable { + + public static final String WORKING = "working"; + + public static final String IMMUTABLE = "immutable"; + + // manage tagKey -> MemChunkGroup + private Map memChunkGroupMap; + + private String status; + + // if the memTable is immutable, the data cannot be deleted directly, and the deleted data needs + // to be recorded in the deletionList + private Set deletionList; + + public MemTable(String status) { + memChunkGroupMap = new HashMap<>(); + this.status = status; + deletionList = new HashSet<>(); + } + + public void put(String tagKey) { + if (this.status.equals(IMMUTABLE)) return; + if (!memChunkGroupMap.containsKey(tagKey)) { + memChunkGroupMap.put(tagKey, new MemChunkGroup()); + } + } + + @Override + public String toString() { + return "MemTable{" + + "memChunkGroupMap=" + + memChunkGroupMap + + ", status='" + + status + + '\'' + + ", deletionList=" + + deletionList + + '}'; + } + + public MemChunkGroup get(String tagKey) { + return memChunkGroupMap.get(tagKey); + } + + public void remove(String tagKey) { + memChunkGroupMap.remove(tagKey); + } + + public boolean isImmutable() { + return status.equals(IMMUTABLE); + } + + public Set getDeletionList() { + return deletionList; + } + + public void setStatus(String status) { + this.status = status; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java new file mode 100644 index 000000000000..1ba77a754b54 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.lsm.context.QueryRequestContext; +import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; + +import java.util.ArrayList; +import java.util.List; + +/** query for MemChunkGroup */ +public class MemChunkGroupQuery extends QueryLevelProcess { + + /** + * get all MemChunks that need to be processed in the current MemChunkGroup + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunks + */ + @Override + public List getChildren(MemChunkGroup memNode, QueryRequestContext context) { + List memChunks = new ArrayList<>(); + String tagValue = (String) context.getKey(); + MemChunk child = memNode.get(tagValue); + if (child != null) memChunks.add(child); + return memChunks; + } + + /** + * the insert method corresponding to the MemChunkGroup node + * + * @param memNode memory node + * @param context query request context + */ + @Override + public void query(MemChunkGroup memNode, QueryRequestContext context) {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java new file mode 100644 index 000000000000..d60be13c8431 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.lsm.context.QueryRequestContext; +import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; + +import org.roaringbitmap.RoaringBitmap; + +import java.util.List; + +/** query for MemChunk */ +public class MemChunkQuery extends QueryLevelProcess { + + /** + * MemChunk is the last layer of memory nodes, no children + * + * @param memNode memory node + * @param context request context + * @return null + */ + @Override + public List getChildren(MemChunk memNode, QueryRequestContext context) { + return null; + } + + /** + * the query method corresponding to the MemChunk node + * + * @param memNode memory node + * @param context query request context + */ + @Override + public void query(MemChunk memNode, QueryRequestContext context) { + RoaringBitmap roaringBitmap = (RoaringBitmap) context.getResult(); + if (roaringBitmap == null) roaringBitmap = new RoaringBitmap(); + RoaringBitmap now = RoaringBitmap.or(roaringBitmap, memNode.getRoaringBitmap()); + context.setResult(now); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java new file mode 100644 index 000000000000..e5ece3a13a99 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.QueryRequestContext; +import org.apache.iotdb.lsm.levelProcess.QueryLevelProcess; + +import org.roaringbitmap.RoaringBitmap; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** query for MemTable */ +public class MemTableQuery extends QueryLevelProcess { + + /** + * get all MemChunkGroups that need to be processed in the current MemTable + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunkGroups + */ + @Override + public List getChildren(MemTable memNode, QueryRequestContext context) { + List memChunkGroups = new ArrayList<>(); + String tagKey = (String) context.getKey(); + MemChunkGroup child = memNode.get(tagKey); + if (child != null) memChunkGroups.add(child); + return memChunkGroups; + } + + /** + * the query method corresponding to the MemTable node + * + * @param memNode memory node + * @param context query request context + */ + @Override + public void query(MemTable memNode, QueryRequestContext context) { + // if the memTable is immutable, we need to delete the id in deletionList in the query result + if (memNode.isImmutable()) { + RoaringBitmap roaringBitmap = (RoaringBitmap) context.getResult(); + Set deletionList = memNode.getDeletionList(); + for (Integer id : deletionList) { + roaringBitmap.remove(id); + } + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java new file mode 100644 index 000000000000..d3dc3a1d3053 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/QueryManager.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.context.QueryRequestContext; +import org.apache.iotdb.lsm.manager.BasicLsmManager; + +/** manage insertion to MemTable */ +public class QueryManager extends BasicLsmManager { + + public QueryManager() { + initLevelProcess(); + } + + /** set the query operation for each layer of memory nodes */ + private void initLevelProcess() { + this.nextLevel(new MemTableQuery()) + .nextLevel(new MemChunkGroupQuery()) + .nextLevel(new MemChunkQuery()); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java new file mode 100644 index 000000000000..5c697e1c77b7 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/recover/RecoverManager.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.recover; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.ITagInvertedIndex; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; +import org.apache.iotdb.lsm.context.DeleteRequestContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; +import org.apache.iotdb.lsm.context.RequestContext; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** for memory structure recovery */ +public class RecoverManager { + private static final Logger logger = LoggerFactory.getLogger(RecoverManager.class); + + private WALManager walManager; + + public RecoverManager(WALManager walManager) { + this.walManager = walManager; + } + + /** + * recover tagInvertedIndex + * + * @param tagInvertedIndex tag inverted index + */ + public void recover(ITagInvertedIndex tagInvertedIndex) { + logger.info("recover tagInvertedIndex"); + while (true) { + RequestContext context = walManager.read(); + switch (context.getType()) { + case INSERT: + tagInvertedIndex.addTags((InsertRequestContext) context); + break; + case DELETE: + tagInvertedIndex.removeTags((DeleteRequestContext) context); + break; + default: + return; + } + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java new file mode 100644 index 000000000000..5d73ed74ae60 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal; + +import org.apache.iotdb.lsm.wal.WALRecord; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +/** represents a record in the wal file */ +public class WALEntry extends WALRecord { + + // can be insertion(1) or deletion(2) + private int type; + + // keys at each level + private List keys; + + // device id + private int deviceID; + + public WALEntry() { + super(); + } + + public WALEntry(int type, List keys, int deviceID) { + super(); + this.type = type; + this.keys = keys; + this.deviceID = deviceID; + } + + /** + * serialize the wal entry + * + * @param buffer byte buffer + */ + @Override + public void serialize(ByteBuffer buffer) { + ReadWriteIOUtils.write(type, buffer); + ReadWriteIOUtils.write(deviceID, buffer); + ReadWriteIOUtils.write(keys.size(), buffer); + for (String key : keys) { + ReadWriteIOUtils.write(key, buffer); + } + } + + /** + * deserialize from DataInputStream + * + * @param stream data input stream + * @throws IOException + */ + @Override + public void deserialize(DataInputStream stream) throws IOException { + this.type = stream.readInt(); + this.deviceID = stream.readInt(); + int length = stream.readInt(); + this.keys = new ArrayList<>(); + for (int i = 0; i < length; i++) { + String key = ReadWriteIOUtils.readString(stream); + keys.add(key); + } + } + + public int getType() { + return type; + } + + public void setType(int type) { + this.type = type; + } + + public List getKeys() { + return keys; + } + + public void setKeys(List keys) { + this.keys = keys; + } + + public int getDeviceID() { + return deviceID; + } + + public void setDeviceID(int deviceID) { + this.deviceID = deviceID; + } + + @Override + public String toString() { + return "WALEntry{" + "type=" + type + ", keys=" + keys + ", deviceID=" + deviceID + '}'; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java new file mode 100644 index 000000000000..e5047a0233b6 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal; + +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaConfig; +import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor; +import org.apache.iotdb.lsm.context.DeleteRequestContext; +import org.apache.iotdb.lsm.context.InsertRequestContext; +import org.apache.iotdb.lsm.context.RequestContext; +import org.apache.iotdb.lsm.wal.WALReader; +import org.apache.iotdb.lsm.wal.WALWriter; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** Manage wal entry writes and reads */ +public class WALManager { + + private static final String WAL_FILE_NAME = "tag_inverted_index.log"; + + private static final int INSERT = 1; + + private static final int DELETE = 2; + + private final TagSchemaConfig tagSchemaConfig = + TagSchemaDescriptor.getInstance().getTagSchemaConfig(); + + private final String schemaDirPath; + + private File walFile; + + // directly use the wal writer that comes with the lsm framework + private WALWriter walWriter; + + // directly use the wal reader that comes with the lsm framework + private WALReader walReader; + + public WALManager(String schemaDirPath) throws IOException { + this.schemaDirPath = schemaDirPath; + initFile(schemaDirPath); + int walBufferSize = tagSchemaConfig.getWalBufferSize(); + walWriter = new WALWriter(walFile, walBufferSize, false); + walReader = new WALReader(walFile, new WALEntry()); + } + + private void initFile(String schemaDirPath) throws IOException { + File schemaDir = new File(schemaDirPath); + schemaDir.mkdirs(); + walFile = new File(this.schemaDirPath, WAL_FILE_NAME); + if (!walFile.exists()) { + walFile.createNewFile(); + } + } + + /** + * handle wal log writes for each request context + * + * @param context request context + * @throws IOException + */ + public synchronized void write(RequestContext context) throws IOException { + switch (context.getType()) { + case INSERT: + process((InsertRequestContext) context); + break; + case DELETE: + process((DeleteRequestContext) context); + break; + default: + break; + } + } + + /** + * for recover + * + * @return request context + */ + public synchronized RequestContext read() { + if (walReader.hasNext()) { + WALEntry walEntry = (WALEntry) walReader.next(); + if (walEntry.getType() == INSERT) { + return generateInsertContext(walEntry); + } + if (walEntry.getType() == DELETE) { + return generateDeleteContext(walEntry); + } + } + return new RequestContext(); + } + + /** + * generate insert context from wal entry + * + * @param walEntry wal entry + * @return insert context + */ + private InsertRequestContext generateInsertContext(WALEntry walEntry) { + InsertRequestContext insertContext = new InsertRequestContext(); + List objects = new ArrayList<>(); + objects.addAll(walEntry.getKeys()); + insertContext.setKeys(objects); + insertContext.setValue(walEntry.getDeviceID()); + insertContext.setRecover(true); + return insertContext; + } + + /** + * generate delete context from wal entry + * + * @param walEntry wal entry + * @return delete context + */ + private DeleteRequestContext generateDeleteContext(WALEntry walEntry) { + DeleteRequestContext deleteContext = + new DeleteRequestContext(walEntry.getDeviceID(), walEntry.getKeys()); + List objects = new ArrayList<>(); + objects.addAll(walEntry.getKeys()); + deleteContext.setKeys(objects); + deleteContext.setValue(walEntry.getDeviceID()); + deleteContext.setRecover(true); + return deleteContext; + } + + /** + * handle wal log writes for each insert context + * + * @param insertContext insert context + * @throws IOException + */ + private void process(InsertRequestContext insertContext) throws IOException { + List objects = insertContext.getKeys(); + List keys = new ArrayList<>(); + for (Object o : objects) { + keys.add((String) o); + } + WALEntry walEntry = new WALEntry(INSERT, keys, (Integer) insertContext.getValue()); + walWriter.write(walEntry); + } + + /** + * handle wal log writes for each delete context + * + * @param deleteContext delete context + * @throws IOException + */ + private void process(DeleteRequestContext deleteContext) throws IOException { + List objects = deleteContext.getKeys(); + List keys = new ArrayList<>(); + for (Object o : objects) { + keys.add((String) o); + } + WALEntry walEntry = new WALEntry(DELETE, keys, (Integer) deleteContext.getValue()); + walWriter.write(walEntry); + } + + @TestOnly + public void close() throws IOException { + walWriter.close(); + walReader.close(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java new file mode 100644 index 000000000000..b6f5402a7169 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.utils; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.metadata.path.MeasurementPath; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +/** process MeasurementPath */ +public class MeasurementPathUtils { + + /** + * generate MeasurementPath + * + * @param devicePath device path + * @param measurement measurement + * @param schemaEntry schema entry + * @param isAligned is aligned + * @return MeasurementPath + * @throws IllegalPathException + */ + public static MeasurementPath generateMeasurementPath( + String devicePath, String measurement, SchemaEntry schemaEntry, boolean isAligned) + throws IllegalPathException { + MeasurementPath measurementPath = + new MeasurementPath( + devicePath, + measurement, + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(isAligned); + return measurementPath; + } + + /** + * generate MeasurementPath + * + * @param diskSchemaEntry disk schema entry + * @return MeasurementPath + * @throws IllegalPathException + */ + public static MeasurementPath generateMeasurementPath(DiskSchemaEntry diskSchemaEntry) + throws IllegalPathException { + MeasurementPath measurementPath = + new MeasurementPath( + new PartialPath(diskSchemaEntry.seriesKey), + new MeasurementSchema( + diskSchemaEntry.measurementName, + TSDataType.deserialize(diskSchemaEntry.type), + TSEncoding.deserialize(diskSchemaEntry.encoding), + CompressionType.deserialize(diskSchemaEntry.compressor))); + measurementPath.setUnderAlignedEntity(diskSchemaEntry.isAligned); + return measurementPath; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java new file mode 100644 index 000000000000..59208fc418b6 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.utils; + +import java.util.Map; +import java.util.TreeMap; + +/** path and tag converter */ +public class PathTagConverterUtils { + + /** + * convert the path of the tree model to the tags of the tag model + * + * @param storageGroupFullPath storage group full path + * @param path path of the tree model + * @return tags of the tag model + */ + public static Map pathToTags(String storageGroupFullPath, String path) { + if (path.length() <= storageGroupFullPath.length()) return new TreeMap<>(); + String devicePath = path.substring(storageGroupFullPath.length() + 1); + String[] tags = devicePath.split("\\."); + Map tagsMap = new TreeMap<>(); + for (int i = 0; i < tags.length; i += 2) { + tagsMap.put(tags[i], tags[i + 1]); + } + return tagsMap; + } + + /** + * convert the tags of the tag model to the path of the tree model + * + * @param storageGroupFullPath storage group full path + * @param tags tags of the tag model + * @return path of the tree model + */ + public static String tagsToPath(String storageGroupFullPath, Map tags) { + StringBuilder stringBuilder = new StringBuilder(storageGroupFullPath); + for (String tagKey : tags.keySet()) { + stringBuilder.append(".").append(tagKey).append(".").append(tags.get(tagKey)); + } + return stringBuilder.toString(); + } + + /** + * generate unique path for paths with the same semantics + * + * @param storageGroupFullPath storage group full path + * @param path path of the tree model + * @return unique path of the tree model + */ + public static String pathToTagsSortPath(String storageGroupFullPath, String path) { + return tagsToPath(storageGroupFullPath, pathToTags(storageGroupFullPath, path)); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java new file mode 100644 index 000000000000..18d4c2266219 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.utils; + +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; + +import java.util.HashMap; + +/** process show timeSeries result */ +public class ShowTimeSeriesResultUtils { + + /** + * generate show timeSeries result + * + * @param sgName storage group name + * @param devicePath device path + * @param measurement measurement + * @param schemaEntry schema entry + * @return ShowTimeSeriesResult + */ + public static ShowTimeSeriesResult generateShowTimeSeriesResult( + String sgName, String devicePath, String measurement, SchemaEntry schemaEntry) { + return new ShowTimeSeriesResult( + devicePath + "." + measurement, + null, + sgName, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>()); + } + + /** + * generate show timeSeries result + * + * @param sgName storage group name + * @param timeSeriesPath timeSeries path + * @param schemaEntry schema entry + * @return ShowTimeSeriesResult + */ + public static ShowTimeSeriesResult generateShowTimeSeriesResult( + String sgName, String timeSeriesPath, SchemaEntry schemaEntry) { + return new ShowTimeSeriesResult( + timeSeriesPath, + null, + sgName, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>()); + } +} diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java new file mode 100644 index 000000000000..26daa70890f1 --- /dev/null +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/deviceidlist/DeviceIDListTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.metadata.tagSchemaRegion.deviceidlist; + +import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class DeviceIDListTest { + + private IDeviceIDList deviceIDList; + + private String storageGroupDirPath; + + private String schemaRegionDirPath; + + private String storageGroupFullPath = "root/testDeviceIDList"; + + private boolean isEnableIDTable; + + private String originalDeviceIDTransformationMethod; + + private String schemaDir; + + private String[] devicePaths = + new String[] { + storageGroupFullPath + ".a.b.c.d1", + storageGroupFullPath + ".a.b.c.d2", + storageGroupFullPath + ".a.b.c.d3", + storageGroupFullPath + ".a.b.c.d4", + storageGroupFullPath + ".a.b.d.d1", + storageGroupFullPath + ".a.b.d.d2", + storageGroupFullPath + ".a.b.e.d1", + storageGroupFullPath + ".a.b.f.d1", + storageGroupFullPath + ".a.b.g.d1", + storageGroupFullPath + ".a.b.h.d1", + }; + + @Before + public void setUp() throws Exception { + schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir(); + isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable(); + originalDeviceIDTransformationMethod = + IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod(); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + storageGroupDirPath = schemaDir + File.separator + storageGroupFullPath; + schemaRegionDirPath = storageGroupDirPath + File.separator + 0; + deviceIDList = new DeviceIDList(schemaRegionDirPath); + } + + @After + public void tearDown() throws Exception { + deviceIDList.clear(); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); + IoTDBDescriptor.getInstance() + .getConfig() + .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); + FileUtils.deleteDirectoryAndEmptyParent(new File(schemaDir)); + } + + @Test + public void testAddandGetDeviceID() { + List deviceIDS = generateTestDeviceIDS(); + for (IDeviceID deviceID : deviceIDS) { + deviceIDList.add(deviceID); + System.out.println(deviceID.toStringID()); + } + for (int i = 0; i < deviceIDS.size(); i++) { + assertEquals(deviceIDS.get(i), deviceIDList.get(i)); + } + } + + @Test + public void testRecover() throws IOException { + List deviceIDS = generateTestDeviceIDS(); + for (IDeviceID deviceID : deviceIDS) { + deviceIDList.add(deviceID); + System.out.println(deviceID.toStringID()); + } + + deviceIDList.clear(); + + deviceIDList = new DeviceIDList(schemaRegionDirPath); + for (int i = 0; i < deviceIDS.size(); i++) { + assertEquals(deviceIDS.get(i), deviceIDList.get(i)); + } + } + + private List generateTestDeviceIDS() { + List deviceIDS = new ArrayList<>(); + for (String devicePath : devicePaths) { + deviceIDS.add(DeviceIDFactory.getInstance().getDeviceID(devicePath)); + } + return deviceIDS; + } +} diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java new file mode 100644 index 000000000000..e8599f4b985c --- /dev/null +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndexTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaDescriptor; +import org.apache.iotdb.tsfile.utils.Pair; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class TagInvertedIndexTest { + private String[][] record = + new String[][] { + {"tag1=q", "tag2=a", "1"}, + {"tag1=q", "tag2=s", "2"}, + {"tag1=q", "tag2=a", "tag3=z", "3"}, + {"tag1=q", "tag3=v", "4"}, + {"tag1=q", "tag2=s", "5"}, + {"tag1=w", "tag2=d", "6"}, + {"tag1=q", "tag2=d", "tag3=e", "7"}, + {"tag1=t", "tag2=g", "8"}, + {"tag1=r", "tag2=d", "9"}, + {"tag1=t", "tag2=f", "10"}, + {"tag1=t", "tag2=h", "11"}, + {"tag1=q", "tag2=a", "tag3=l", "12"}, + {"tag1=y", "tag2=j", "13"}, + {"tag1=u", "tag2=k", "14"}, + {"tag1=q", "tag2=a", "tag3=x", "15"}, + {"tag1=q", "tag2=a", "tag4=z", "16"}, + {"tag1=y", "tag2=a", "tag4=z", "17"}, + {"tag1=q", "tag2=b", "tag3=x", "18"}, + }; + + private int numOfDeviceIdsInMemTable; + + private TagInvertedIndex tagInvertedIndex; + + private String storageGroupDirPath; + + private String schemaRegionDirPath; + + private String storageGroupFullPath = "root/testTagIndex"; + + private String schemaDir; + + @Before + public void setUp() throws Exception { + numOfDeviceIdsInMemTable = + TagSchemaDescriptor.getInstance().getTagSchemaConfig().getNumOfDeviceIdsInMemTable(); + TagSchemaDescriptor.getInstance().getTagSchemaConfig().setNumOfDeviceIdsInMemTable(3); + schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir(); + storageGroupDirPath = schemaDir + File.separator + storageGroupFullPath; + schemaRegionDirPath = storageGroupDirPath + File.separator + 0; + tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath); + } + + @After + public void tearDown() throws Exception { + TagSchemaDescriptor.getInstance() + .getTagSchemaConfig() + .setNumOfDeviceIdsInMemTable(numOfDeviceIdsInMemTable); + tagInvertedIndex.clear(); + tagInvertedIndex = null; + FileUtils.deleteDirectoryAndEmptyParent(new File(schemaDir)); + } + + public void addTags() { + List, Integer>> records = generateTags(); + for (Pair, Integer> pair : records) { + tagInvertedIndex.addTags(pair.left, pair.right); + } + } + + public void removeTags() { + Pair, Integer> tags = generateTag(record[0]); + tagInvertedIndex.removeTags(tags.left, tags.right); + tags = generateTag(record[1]); + tagInvertedIndex.removeTags(tags.left, tags.right); + tags = generateTag(record[3]); + tagInvertedIndex.removeTags(tags.left, tags.right); + tags = generateTag(record[11]); + tagInvertedIndex.removeTags(tags.left, tags.right); + } + + @Test + public void getMatchedIDs() { + addTags(); + Map tags1 = new HashMap<>(); + tags1.put("tag1", "q"); + + Map tags2 = new HashMap<>(); + tags2.put("tag1", "q"); + tags2.put("tag2", "a"); + + List ids = tagInvertedIndex.getMatchedIDs(tags1); + List verify = Arrays.asList(1, 2, 3, 4, 5, 7, 12, 15, 16, 18); + assertEquals(verify, ids); + + ids = tagInvertedIndex.getMatchedIDs(tags2); + verify = Arrays.asList(1, 3, 12, 15, 16); + assertEquals(verify, ids); + + removeTags(); + + ids = tagInvertedIndex.getMatchedIDs(tags1); + verify = Arrays.asList(3, 5, 7, 15, 16, 18); + assertEquals(verify, ids); + + ids = tagInvertedIndex.getMatchedIDs(tags2); + verify = Arrays.asList(3, 15, 16); + assertEquals(verify, ids); + } + + @Test + public void testRecover() throws IOException { + Map tags1 = new HashMap<>(); + tags1.put("tag1", "q"); + + Map tags2 = new HashMap<>(); + tags2.put("tag1", "q"); + tags2.put("tag2", "a"); + addTags(); + removeTags(); + + tagInvertedIndex.clear(); + tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath); + + List ids = tagInvertedIndex.getMatchedIDs(tags1); + List verify = Arrays.asList(3, 5, 7, 15, 16, 18); + assertEquals(verify, ids); + + ids = tagInvertedIndex.getMatchedIDs(tags2); + verify = Arrays.asList(3, 15, 16); + assertEquals(verify, ids); + } + + private List, Integer>> generateTags() { + List, Integer>> pairs = new ArrayList<>(); + for (String[] strings : record) { + pairs.add(generateTag(strings)); + } + return pairs; + } + + private Pair, Integer> generateTag(String[] strings) { + Map tags = new HashMap<>(); + int i = 0; + for (; i < strings.length - 1; i++) { + String[] str = strings[i].split("="); + tags.put(str[0], str[1]); + } + Pair, Integer> pair = new Pair<>(tags, Integer.valueOf(strings[i])); + return pair; + } +} diff --git a/server/src/assembly/resources/conf/iotdb-datanode.properties b/server/src/assembly/resources/conf/iotdb-datanode.properties index 6a275d7bf3e2..8ad232903353 100644 --- a/server/src/assembly/resources/conf/iotdb-datanode.properties +++ b/server/src/assembly/resources/conf/iotdb-datanode.properties @@ -1071,10 +1071,23 @@ timestamp_precision=ms # Datatype: float # group_by_fill_cache_size_in_mb=1.0 +#################### +### IDTable Configuration +#################### +# Datatype: boolean +# enable_id_table=false + +# Datatype: boolean +# enable_id_table_log_file=false + +# Choose the device id transformation method. The value could be Plain and SHA256. If the provided value doesn't match any pre-defined value, Plain will be used as default. +# Datatype: string +# device_id_transformation_method=Plain + #################### ### Schema Engine Configuration #################### -# Choose the mode of schema engine. The value could be Memory,Schema_File and Rocksdb_based. If the provided value doesn't match any pre-defined value, Memory mode will be used as default. +# Choose the mode of schema engine. The value could be Memory,Schema_File,Rocksdb_based and Tag. If the provided value doesn't match any pre-defined value, Memory mode will be used as default. # Datatype: string # schema_engine_mode=Memory diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java index e0e21ddbfaa6..f6bd30867cf1 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java @@ -74,6 +74,10 @@ public DiskSchemaEntry( this.isAligned = isAligned; } + public String getDevicePath() { + return seriesKey.substring(0, seriesKey.length() - measurementName.length() - 1); + } + public int serialize(OutputStream outputStream) throws IOException { int byteLen = 0; byteLen += ReadWriteIOUtils.write(deviceID, outputStream); diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java index 358d37f7b8bc..c6762738695c 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngine.java @@ -323,6 +323,12 @@ private ISchemaRegion createSchemaRegionWithoutExistenceCheck( new RSchemaRegionLoader() .loadRSchemaRegion(storageGroup, schemaRegionId, storageGroupMNode); break; + case Tag: + schemaRegion = + new TagSchemaRegionLoader() + .loadTagSchemaRegion( + storageGroup, schemaRegionId, storageGroupMNode, seriesNumerLimiter); + break; default: throw new UnsupportedOperationException( String.format( diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngineMode.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngineMode.java index 9147b9374c01..6b3270e87612 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngineMode.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaEngineMode.java @@ -22,5 +22,6 @@ public enum SchemaEngineMode { Memory, Schema_File, - Rocksdb_based + Rocksdb_based, + Tag } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java new file mode 100644 index 000000000000..694ffe7929d6 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/TagSchemaRegionLoader.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.schemaregion; + +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; +import org.apache.iotdb.external.api.ISeriesNumerLimiter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.LinkedList; +import java.util.List; + +public class TagSchemaRegionLoader { + private static final Logger LOGGER = LoggerFactory.getLogger(TagSchemaRegionLoader.class); + private static URLClassLoader urlClassLoader = null; + private static final String TAG_SCHEMA_REGION_CLASS_NAME = + "org.apache.iotdb.db.metadata.tagSchemaRegion.TagSchemaRegion"; + private static final String LIB_PATH = + ".." + File.separator + "lib" + File.separator + "tag-schema-region" + File.separator; + + public TagSchemaRegionLoader() {} + + /** + * Load the jar files for TagSchemaRegion and create an instance of it. The jar files should be + * located in "../lib/tag-schema-region". If jar files cannot be found, the function will return + * null. + * + * @param storageGroup + * @param schemaRegionId + * @param node + * @return + */ + public ISchemaRegion loadTagSchemaRegion( + PartialPath storageGroup, + SchemaRegionId schemaRegionId, + IStorageGroupMNode node, + ISeriesNumerLimiter seriesNumerLimiter) { + ISchemaRegion region = null; + LOGGER.info("Creating instance for schema-engine-tag"); + try { + loadTagSchemaRegionJar(); + Class classForTagSchemaRegion = urlClassLoader.loadClass(TAG_SCHEMA_REGION_CLASS_NAME); + Constructor constructor = + classForTagSchemaRegion.getConstructor( + PartialPath.class, + SchemaRegionId.class, + IStorageGroupMNode.class, + ISeriesNumerLimiter.class); + + region = + (ISchemaRegion) + constructor.newInstance(storageGroup, schemaRegionId, node, seriesNumerLimiter); + } catch (ClassNotFoundException + | NoSuchMethodException + | InvocationTargetException + | InstantiationException + | IllegalAccessException + | MalformedURLException + | RuntimeException e) { + LOGGER.error("Cannot initialize TagSchemaRegion", e); + return null; + } + return region; + } + + /** + * Load the jar files for TagSchemaRegion. The jar files should be located in directory + * "../lib/tag-schema-region". If the jar files have been loaded, it will do nothing. + */ + private void loadTagSchemaRegionJar() throws MalformedURLException { + LOGGER.info("Loading jar for schema-engine-tag"); + if (urlClassLoader == null) { + File[] jars = new File(LIB_PATH).listFiles(); + if (jars == null) { + throw new RuntimeException( + String.format("Cannot get jars from %s", new File(LIB_PATH).getAbsolutePath())); + } + List dependentJars = new LinkedList<>(); + for (File jar : jars) { + if (jar.getName().endsWith(".jar")) { + dependentJars.add(new URL("file:" + jar.getAbsolutePath())); + } + } + LOGGER.info("load jars: " + dependentJars); + urlClassLoader = new URLClassLoader(dependentJars.toArray(new URL[] {})); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java index ae5e7b97b571..112a9137f5e9 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/ClusterSchemaTree.java @@ -22,7 +22,6 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.metadata.path.MeasurementPath; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaEntityNode; import org.apache.iotdb.db.mpp.common.schematree.node.SchemaInternalNode; @@ -30,6 +29,7 @@ import org.apache.iotdb.db.mpp.common.schematree.node.SchemaNode; import org.apache.iotdb.db.mpp.common.schematree.visitor.SchemaTreeDeviceVisitor; import org.apache.iotdb.db.mpp.common.schematree.visitor.SchemaTreeMeasurementVisitor; +import org.apache.iotdb.db.mpp.common.schematree.visitor.SchemaTreeVisitorFactory; import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; @@ -79,12 +79,7 @@ public Pair, Integer> searchMeasurementPaths( @Override public Pair, Integer> searchMeasurementPaths(PartialPath pathPattern) { SchemaTreeMeasurementVisitor visitor = - new SchemaTreeMeasurementVisitor( - root, - pathPattern, - IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, - 0, - false); + SchemaTreeVisitorFactory.getInstance().getSchemaTreeMeasurementVisitor(root, pathPattern); return new Pair<>(visitor.getAllResult(), visitor.getNextOffset()); } @@ -101,13 +96,16 @@ public List getAllMeasurement() { */ @Override public List getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) { - SchemaTreeDeviceVisitor visitor = new SchemaTreeDeviceVisitor(root, pathPattern, isPrefixMatch); + SchemaTreeDeviceVisitor visitor = + SchemaTreeVisitorFactory.getInstance() + .getSchemaTreeDeviceVisitor(root, pathPattern, isPrefixMatch); return visitor.getAllResult(); } @Override public List getMatchedDevices(PartialPath pathPattern) { - SchemaTreeDeviceVisitor visitor = new SchemaTreeDeviceVisitor(root, pathPattern, false); + SchemaTreeDeviceVisitor visitor = + SchemaTreeVisitorFactory.getInstance().getSchemaTreeDeviceVisitor(root, pathPattern, false); return visitor.getAllResult(); } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/visitor/SchemaTreeVisitorFactory.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/visitor/SchemaTreeVisitorFactory.java new file mode 100644 index 000000000000..cc669ac6f717 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/schematree/visitor/SchemaTreeVisitorFactory.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.mpp.common.schematree.visitor; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.schemaregion.SchemaEngineMode; +import org.apache.iotdb.db.mpp.common.schematree.node.SchemaNode; + +import static org.apache.iotdb.db.metadata.MetadataConstant.ALL_MATCH_PATTERN; + +/** + * Generate corresponding SchemaTreeVisitor for different types of schema regions, such as Memory, + * Schema_File, Rocksdb_based, and Tag + */ +public class SchemaTreeVisitorFactory { + + private SchemaTreeVisitorFactory() {} + + public static SchemaTreeVisitorFactory getInstance() { + return SchemaTreeVisitorFactoryHolder.INSTANCE; + } + + /** + * generate corresponding SchemaTreeDeviceVisitor for different types of schema regions + * + * @param schemaNode schema node + * @param pathPattern can be a pattern or a full path of timeseries. + * @param isPrefixMatch + * @return schemaTreeDeviceVisitor + */ + public SchemaTreeDeviceVisitor getSchemaTreeDeviceVisitor( + SchemaNode schemaNode, PartialPath pathPattern, boolean isPrefixMatch) { + SchemaTreeDeviceVisitor visitor; + switch (SchemaEngineMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + case Memory: + case Schema_File: + visitor = new SchemaTreeDeviceVisitor(schemaNode, pathPattern, isPrefixMatch); + break; + default: + visitor = new SchemaTreeDeviceVisitor(schemaNode, ALL_MATCH_PATTERN, isPrefixMatch); + break; + } + return visitor; + } + + /** + * generate corresponding SchemaTreeMeasurementVisitor for different types of schema regions + * + * @param schemaNode schema node + * @param pathPattern can be a pattern or a full path of timeseries. + * @return schemaTreeMeasurementVisitor + */ + public SchemaTreeMeasurementVisitor getSchemaTreeMeasurementVisitor( + SchemaNode schemaNode, PartialPath pathPattern) { + SchemaTreeMeasurementVisitor visitor; + switch (SchemaEngineMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + case Memory: + case Schema_File: + visitor = generateSchemaTreeMeasurementVisitor(schemaNode, pathPattern); + break; + case Tag: + if (pathPattern.getFullPath().contains(".**")) { + visitor = + generateSchemaTreeMeasurementVisitor( + schemaNode, ALL_MATCH_PATTERN.concatNode(pathPattern.getMeasurement())); + } else { + visitor = generateSchemaTreeMeasurementVisitor(schemaNode, pathPattern); + } + break; + default: + visitor = generateSchemaTreeMeasurementVisitor(schemaNode, ALL_MATCH_PATTERN); + break; + } + return visitor; + } + + private SchemaTreeMeasurementVisitor generateSchemaTreeMeasurementVisitor( + SchemaNode schemaNode, PartialPath pathPattern) { + return new SchemaTreeMeasurementVisitor( + schemaNode, + pathPattern, + IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1, + 0, + false); + } + + private static class SchemaTreeVisitorFactoryHolder { + private static final SchemaTreeVisitorFactory INSTANCE = new SchemaTreeVisitorFactory(); + + private SchemaTreeVisitorFactoryHolder() {} + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java index ac7c1ce9842e..9f5aea798383 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.protocol.influxdb.meta.AbstractInfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.IInfluxDBMetaManager; import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; import org.apache.iotdb.db.utils.DataTypeUtils; import org.apache.iotdb.db.utils.ParameterUtils; @@ -59,7 +59,7 @@ public IoTDBPoint( } public IoTDBPoint( - String database, Point point, AbstractInfluxDBMetaManager metaManager, long sessionID) { + String database, Point point, IInfluxDBMetaManager influxDBMetaManager, long sessionID) { String measurement = null; Map tags = new HashMap<>(); Map fields = new HashMap<>(); @@ -105,7 +105,8 @@ public IoTDBPoint( } ParameterUtils.checkNonEmptyString(database, "database"); ParameterUtils.checkNonEmptyString(measurement, "measurement name"); - String path = metaManager.generatePath(database, measurement, tags, sessionID); + String path = + influxDBMetaManager.generatePath(database, measurement, tags, fields.keySet(), sessionID); List measurements = new ArrayList<>(); List types = new ArrayList<>(); List values = new ArrayList<>(); diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java index 78899f7e34bb..10bf4dced928 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java @@ -29,7 +29,7 @@ import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; import org.apache.iotdb.db.protocol.influxdb.function.aggregator.InfluxAggregator; import org.apache.iotdb.db.protocol.influxdb.function.selector.InfluxSelector; -import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; import org.apache.iotdb.db.protocol.influxdb.operator.InfluxQueryOperator; import org.apache.iotdb.db.protocol.influxdb.operator.InfluxSelectComponent; import org.apache.iotdb.db.protocol.influxdb.util.FilterUtils; @@ -40,7 +40,6 @@ import org.apache.iotdb.db.qp.logical.Operator; import org.apache.iotdb.db.qp.logical.crud.BasicFunctionOperator; import org.apache.iotdb.db.qp.logical.crud.FilterOperator; -import org.apache.iotdb.db.service.basic.ServiceProvider; import org.apache.iotdb.protocol.influxdb.rpc.thrift.InfluxQueryResultRsp; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -57,30 +56,24 @@ public abstract class AbstractQueryHandler { - abstract Map getFieldOrders( - String database, String measurement, ServiceProvider serviceProvider, long sessionId); - abstract InfluxFunctionValue updateByIoTDBFunc( - InfluxFunction function, ServiceProvider serviceProvider, String path, long sessionid); + String database, String measurement, InfluxFunction function, long sessionid); abstract QueryResult queryByConditions( String querySql, String database, String measurement, - ServiceProvider serviceProvider, + Map tagOrders, Map fieldOrders, long sessionId) throws AuthException; public final InfluxQueryResultRsp queryInfluxDB( - String database, - InfluxQueryOperator queryOperator, - long sessionId, - ServiceProvider serviceProvider) { + String database, InfluxQueryOperator queryOperator, long sessionId) { String measurement = queryOperator.getFromComponent().getPrefixPaths().get(0).getFullPath(); // The list of fields under the current measurement and the order of the specified rules Map fieldOrders = - getFieldOrders(database, measurement, serviceProvider, sessionId); + InfluxDBMetaManagerFactory.getInstance().getFieldOrders(database, measurement, sessionId); QueryResult queryResult; InfluxQueryResultRsp tsQueryResultRsp = new InfluxQueryResultRsp(); try { @@ -96,7 +89,6 @@ public final InfluxQueryResultRsp queryInfluxDB( : null, database, measurement, - serviceProvider, fieldOrders, sessionId); // step2 : select filter @@ -106,11 +98,7 @@ public final InfluxQueryResultRsp queryInfluxDB( else { queryResult = queryFuncWithoutFilter( - queryOperator.getSelectComponent(), - database, - measurement, - serviceProvider, - sessionId); + queryOperator.getSelectComponent(), database, measurement, sessionId); } return tsQueryResultRsp .setResultJsonString(JacksonUtils.bean2Json(queryResult)) @@ -274,18 +262,14 @@ else if (selectComponent.isHasCommonQuery()) { * @param selectComponent select data to query * @return select query result */ - public final QueryResult queryFuncWithoutFilter( - InfluxSelectComponent selectComponent, - String database, - String measurement, - ServiceProvider serviceProvider, - long sessionid) { + public QueryResult queryFuncWithoutFilter( + InfluxSelectComponent selectComponent, String database, String measurement, long sessionid) { // columns List columns = new ArrayList<>(); columns.add(InfluxSQLConstant.RESERVED_TIME); List functions = new ArrayList<>(); - String path = "root." + database + "." + measurement; + for (ResultColumn resultColumn : selectComponent.getResultColumns()) { Expression expression = resultColumn.getExpression(); if (expression instanceof FunctionExpression) { @@ -300,7 +284,7 @@ public final QueryResult queryFuncWithoutFilter( List> values = new ArrayList<>(); for (InfluxFunction function : functions) { InfluxFunctionValue functionValue = - updateByIoTDBFunc(function, serviceProvider, path, sessionid); + updateByIoTDBFunc(database, measurement, function, sessionid); // InfluxFunctionValue functionValue = function.calculateByIoTDBFunc(); if (value.size() == 0) { value.add(functionValue.getTimestamp()); @@ -330,40 +314,33 @@ public QueryResult queryExpr( FilterOperator operator, String database, String measurement, - ServiceProvider serviceProvider, Map fieldOrders, Long sessionId) throws AuthException { if (operator == null) { List expressions = new ArrayList<>(); - return queryByConditions( - expressions, database, measurement, serviceProvider, fieldOrders, sessionId); + return queryByConditions(expressions, database, measurement, fieldOrders, sessionId); } else if (operator instanceof BasicFunctionOperator) { List iExpressions = new ArrayList<>(); iExpressions.add(getIExpressionForBasicFunctionOperator((BasicFunctionOperator) operator)); - return queryByConditions( - iExpressions, database, measurement, serviceProvider, fieldOrders, sessionId); + return queryByConditions(iExpressions, database, measurement, fieldOrders, sessionId); } else { FilterOperator leftOperator = operator.getChildren().get(0); FilterOperator rightOperator = operator.getChildren().get(1); if (operator.getFilterType() == FilterConstant.FilterType.KW_OR) { return QueryResultUtils.orQueryResultProcess( - queryExpr(leftOperator, database, measurement, serviceProvider, fieldOrders, sessionId), - queryExpr( - rightOperator, database, measurement, serviceProvider, fieldOrders, sessionId)); + queryExpr(leftOperator, database, measurement, fieldOrders, sessionId), + queryExpr(rightOperator, database, measurement, fieldOrders, sessionId)); } else if (operator.getFilterType() == FilterConstant.FilterType.KW_AND) { if (canMergeOperator(leftOperator) && canMergeOperator(rightOperator)) { List iExpressions1 = getIExpressionByFilterOperatorOperator(leftOperator); List iExpressions2 = getIExpressionByFilterOperatorOperator(rightOperator); iExpressions1.addAll(iExpressions2); - return queryByConditions( - iExpressions1, database, measurement, serviceProvider, fieldOrders, sessionId); + return queryByConditions(iExpressions1, database, measurement, fieldOrders, sessionId); } else { return QueryResultUtils.andQueryResultProcess( - queryExpr( - leftOperator, database, measurement, serviceProvider, fieldOrders, sessionId), - queryExpr( - rightOperator, database, measurement, serviceProvider, fieldOrders, sessionId)); + queryExpr(leftOperator, database, measurement, fieldOrders, sessionId), + queryExpr(rightOperator, database, measurement, fieldOrders, sessionId)); } } } @@ -376,11 +353,10 @@ public QueryResult queryExpr( * @param expressions list of conditions, including tag and field condition * @return returns the results of the influxdb query */ - private QueryResult queryByConditions( + public QueryResult queryByConditions( List expressions, String database, String measurement, - ServiceProvider serviceProvider, Map fieldOrders, Long sessionId) throws AuthException { @@ -390,7 +366,8 @@ private QueryResult queryByConditions( List fieldExpressions = new ArrayList<>(); // maximum number of tags in the current query criteria int currentQueryMaxTagNum = 0; - Map tagOrders = InfluxDBMetaManager.getTagOrders(database, measurement); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, sessionId); for (IExpression expression : expressions) { SingleSeriesExpression singleSeriesExpression = ((SingleSeriesExpression) expression); // the current condition is in tag @@ -445,8 +422,7 @@ private QueryResult queryByConditions( realQuerySql += " where " + realIotDBCondition; } realQuerySql += " align by device"; - return queryByConditions( - realQuerySql, database, measurement, serviceProvider, fieldOrders, sessionId); + return queryByConditions(realQuerySql, database, measurement, null, fieldOrders, sessionId); } /** diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java index ee8d0db5c06b..3d536df10a32 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java @@ -18,77 +18,31 @@ */ package org.apache.iotdb.db.protocol.influxdb.handler; -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.db.protocol.influxdb.constant.InfluxConstant; import org.apache.iotdb.db.protocol.influxdb.constant.InfluxSQLConstant; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunction; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; -import org.apache.iotdb.db.protocol.influxdb.meta.NewInfluxDBMetaManager; import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; -import org.apache.iotdb.db.service.basic.ServiceProvider; import org.apache.iotdb.db.service.thrift.impl.NewInfluxDBServiceImpl; -import org.apache.iotdb.rpc.TSStatusCode; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; -import org.influxdb.InfluxDBException; import org.influxdb.dto.QueryResult; -import java.util.HashMap; import java.util.List; import java.util.Map; public class NewQueryHandler extends AbstractQueryHandler { - public static TSExecuteStatementResp executeStatement(String sql, long sessionId) { - TSExecuteStatementReq tsExecuteStatementReq = new TSExecuteStatementReq(); - tsExecuteStatementReq.setStatement(sql); - tsExecuteStatementReq.setSessionId(sessionId); - tsExecuteStatementReq.setStatementId( - NewInfluxDBServiceImpl.getClientRPCService().requestStatementId(sessionId)); - tsExecuteStatementReq.setFetchSize(InfluxConstant.DEFAULT_FETCH_SIZE); - TSExecuteStatementResp executeStatementResp = - NewInfluxDBServiceImpl.getClientRPCService().executeStatement(tsExecuteStatementReq); - TSStatus tsStatus = executeStatementResp.getStatus(); - if (tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - throw new InfluxDBException(tsStatus.getMessage()); - } - return executeStatementResp; - } - - @Override - public Map getFieldOrders( - String database, String measurement, ServiceProvider serviceProvider, long sessionID) { - Map fieldOrders = new HashMap<>(); - String showTimeseriesSql = "show timeseries root." + database + '.' + measurement + ".**"; - TSExecuteStatementResp executeStatementResp = executeStatement(showTimeseriesSql, sessionID); - List paths = QueryResultUtils.getFullPaths(executeStatementResp); - Map tagOrders = NewInfluxDBMetaManager.getTagOrders(database, measurement); - int tagOrderNums = tagOrders.size(); - int fieldNums = 0; - for (String path : paths) { - String filed = StringUtils.getFieldByPath(path); - if (!fieldOrders.containsKey(filed)) { - // The corresponding order of fields is 1 + tagNum (the first is timestamp, then all tags, - // and finally all fields) - fieldOrders.put(filed, tagOrderNums + fieldNums + 1); - fieldNums++; - } - } - return fieldOrders; - } - - @Override - public InfluxFunctionValue updateByIoTDBFunc( - InfluxFunction function, ServiceProvider serviceProvider, String path, long sessionid) { + public final InfluxFunctionValue updateByIoTDBFunc( + String path, InfluxFunction function, long sessionid) { switch (function.getFunctionName()) { case InfluxSQLConstant.COUNT: { String functionSql = StringUtils.generateFunctionSql( function.getFunctionName(), function.getParmaName(), path); - TSExecuteStatementResp tsExecuteStatementResp = executeStatement(functionSql, sessionid); + TSExecuteStatementResp tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSql, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { @@ -101,7 +55,7 @@ public InfluxFunctionValue updateByIoTDBFunc( String functionSqlCount = StringUtils.generateFunctionSql("count", function.getParmaName(), path); TSExecuteStatementResp tsExecuteStatementResp = - executeStatement(functionSqlCount, sessionid); + NewInfluxDBServiceImpl.executeStatement(functionSqlCount, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { @@ -109,7 +63,8 @@ public InfluxFunctionValue updateByIoTDBFunc( } String functionSqlSum = StringUtils.generateFunctionSql("sum", function.getParmaName(), path); - tsExecuteStatementResp = executeStatement(functionSqlSum, sessionid); + tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSqlSum, sessionid); list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { function.updateValueIoTDBFunc(null, influxFunctionValue); @@ -120,7 +75,8 @@ public InfluxFunctionValue updateByIoTDBFunc( { String functionSql = StringUtils.generateFunctionSql("sum", function.getParmaName(), path); - TSExecuteStatementResp tsExecuteStatementResp = executeStatement(functionSql, sessionid); + TSExecuteStatementResp tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSql, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { @@ -142,7 +98,8 @@ public InfluxFunctionValue updateByIoTDBFunc( StringUtils.generateFunctionSql("last_value", function.getParmaName(), path); functionName = "last_value"; } - TSExecuteStatementResp tsExecuteStatementResp = executeStatement(functionSql, sessionid); + TSExecuteStatementResp tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSql, sessionid); Map map = QueryResultUtils.getColumnNameAndValue(tsExecuteStatementResp); for (String colume : map.keySet()) { Object o = map.get(colume); @@ -152,7 +109,8 @@ public InfluxFunctionValue updateByIoTDBFunc( String.format( "select %s from %s where %s=%s", function.getParmaName(), devicePath, fullPath, o); - TSExecuteStatementResp resp = executeStatement(specificSql, sessionid); + TSExecuteStatementResp resp = + NewInfluxDBServiceImpl.executeStatement(specificSql, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(resp); for (InfluxFunctionValue influxFunctionValue : list) { function.updateValueIoTDBFunc(influxFunctionValue); @@ -171,7 +129,8 @@ public InfluxFunctionValue updateByIoTDBFunc( functionSql = StringUtils.generateFunctionSql("min_value", function.getParmaName(), path); } - TSExecuteStatementResp tsExecuteStatementResp = executeStatement(functionSql, sessionid); + TSExecuteStatementResp tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSql, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { @@ -185,15 +144,23 @@ public InfluxFunctionValue updateByIoTDBFunc( return function.calculateByIoTDBFunc(); } + @Override + public InfluxFunctionValue updateByIoTDBFunc( + String database, String measurement, InfluxFunction function, long sessionid) { + String path = "root." + database + "." + measurement; + return updateByIoTDBFunc(path, function, sessionid); + } + @Override public QueryResult queryByConditions( String querySql, String database, String measurement, - ServiceProvider serviceProvider, + Map tagOrders, Map fieldOrders, long sessionId) { - TSExecuteStatementResp executeStatementResp = executeStatement(querySql, sessionId); + TSExecuteStatementResp executeStatementResp = + NewInfluxDBServiceImpl.executeStatement(querySql, sessionId); return QueryResultUtils.iotdbResultConvertInfluxResult( executeStatementResp, database, measurement, fieldOrders); } diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java index b58b65f6bf5c..3a895bcb9a2b 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java @@ -27,14 +27,13 @@ import org.apache.iotdb.db.protocol.influxdb.constant.InfluxSQLConstant; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunction; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; -import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManager; import org.apache.iotdb.db.protocol.influxdb.util.FieldUtils; import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; -import org.apache.iotdb.db.qp.physical.PhysicalPlan; import org.apache.iotdb.db.qp.physical.crud.QueryPlan; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.control.SessionManager; +import org.apache.iotdb.db.service.IoTDB; import org.apache.iotdb.db.service.basic.ServiceProvider; import org.apache.iotdb.tsfile.exception.filter.QueryFilterOptimizationException; import org.apache.iotdb.tsfile.read.common.Field; @@ -49,62 +48,17 @@ import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; public class QueryHandler extends AbstractQueryHandler { - @Override - public Map getFieldOrders( - String database, String measurement, ServiceProvider serviceProvider, long sessionID) { - Map fieldOrders = new HashMap<>(); - long queryId = ServiceProvider.SESSION_MANAGER.requestQueryId(true); - try { - String showTimeseriesSql = "show timeseries root." + database + '.' + measurement + ".**"; - PhysicalPlan physicalPlan = - serviceProvider.getPlanner().parseSQLToPhysicalPlan(showTimeseriesSql); - QueryContext queryContext = - serviceProvider.genQueryContext( - queryId, - true, - System.currentTimeMillis(), - showTimeseriesSql, - InfluxConstant.DEFAULT_CONNECTION_TIMEOUT_MS); - QueryDataSet queryDataSet = - serviceProvider.createQueryDataSet( - queryContext, physicalPlan, InfluxConstant.DEFAULT_FETCH_SIZE); - int fieldNums = 0; - Map tagOrders = InfluxDBMetaManager.getTagOrders(database, measurement); - int tagOrderNums = tagOrders.size(); - while (queryDataSet.hasNext()) { - List fields = queryDataSet.next().getFields(); - String filed = StringUtils.getFieldByPath(fields.get(0).getStringValue()); - if (!fieldOrders.containsKey(filed)) { - // The corresponding order of fields is 1 + tagNum (the first is timestamp, then all tags, - // and finally all fields) - fieldOrders.put(filed, tagOrderNums + fieldNums + 1); - fieldNums++; - } - } - } catch (QueryProcessException - | TException - | StorageEngineException - | SQLException - | IOException - | InterruptedException - | QueryFilterOptimizationException - | MetadataException e) { - throw new InfluxDBException(e.getMessage()); - } finally { - ServiceProvider.SESSION_MANAGER.releaseQueryResourceNoExceptions(queryId); - } - return fieldOrders; - } + ServiceProvider serviceProvider = IoTDB.serviceProvider; @Override public InfluxFunctionValue updateByIoTDBFunc( - InfluxFunction function, ServiceProvider serviceProvider, String path, long sessionid) { + String database, String measurement, InfluxFunction function, long sessionid) { + String path = "root." + database + "." + measurement; switch (function.getFunctionName()) { case InfluxSQLConstant.COUNT: { @@ -481,7 +435,7 @@ public QueryResult queryByConditions( String querySql, String database, String measurement, - ServiceProvider serviceProvider, + Map tagOrders, Map fieldOrders, long sessionId) throws AuthException { diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java new file mode 100644 index 000000000000..bfdf413a9c30 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.handler; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.schemaregion.SchemaEngineMode; +import org.apache.iotdb.db.service.thrift.impl.ClientRPCServiceImpl; + +public class QueryHandlerFactory { + public static AbstractQueryHandler getInstance() { + if (IoTDBDescriptor.getInstance() + .getConfig() + .getRpcImplClassName() + .equals(ClientRPCServiceImpl.class.getName())) { + switch (SchemaEngineMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + case Tag: + return new TagQueryHandler(); + default: + return new NewQueryHandler(); + } + } else { + return new QueryHandler(); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java new file mode 100644 index 000000000000..5344bba68c90 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.handler; + +import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunction; +import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; +import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; +import org.apache.iotdb.db.protocol.influxdb.util.FilterUtils; +import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; +import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; +import org.apache.iotdb.db.service.thrift.impl.NewInfluxDBServiceImpl; +import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; +import org.apache.iotdb.tsfile.read.expression.IExpression; +import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression; + +import org.influxdb.dto.QueryResult; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** use in tag schema region */ +public class TagQueryHandler extends NewQueryHandler { + + @Override + public InfluxFunctionValue updateByIoTDBFunc( + String database, String measurement, InfluxFunction function, long sessionid) { + String path = "root." + database + ".measurement." + measurement; + return updateByIoTDBFunc(path, function, sessionid); + } + + @Override + public QueryResult queryByConditions( + String querySql, + String database, + String measurement, + Map tagOrders, + Map fieldOrders, + long sessionId) { + TSExecuteStatementResp executeStatementResp = + NewInfluxDBServiceImpl.executeStatement(querySql, sessionId); + return QueryResultUtils.iotdbResultConvertInfluxResult( + executeStatementResp, database, measurement, tagOrders, fieldOrders); + } + + @Override + public QueryResult queryByConditions( + List expressions, + String database, + String measurement, + Map fieldOrders, + Long sessionId) { + List fieldExpressions = new ArrayList<>(); + List tagExpressions = new ArrayList<>(); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, sessionId); + for (IExpression expression : expressions) { + SingleSeriesExpression singleSeriesExpression = ((SingleSeriesExpression) expression); + // the current condition is in tag + if (tagOrders.containsKey(singleSeriesExpression.getSeriesPath().getFullPath())) { + tagExpressions.add(singleSeriesExpression); + } else { + fieldExpressions.add(singleSeriesExpression); + } + } + // construct the actual query path + StringBuilder curQueryPath = + new StringBuilder("root." + database + ".measurement." + measurement); + for (SingleSeriesExpression singleSeriesExpression : tagExpressions) { + String tagKey = singleSeriesExpression.getSeriesPath().getFullPath(); + String tagValue = + StringUtils.removeQuotation( + FilterUtils.getFilterStringValue(singleSeriesExpression.getFilter())); + curQueryPath.append(".").append(tagKey).append(".").append(tagValue); + } + curQueryPath.append(".**"); + + // construct actual query condition + StringBuilder realIotDBCondition = new StringBuilder(); + for (int i = 0; i < fieldExpressions.size(); i++) { + SingleSeriesExpression singleSeriesExpression = fieldExpressions.get(i); + if (i != 0) { + realIotDBCondition.append(" and "); + } + realIotDBCondition + .append(singleSeriesExpression.getSeriesPath().getFullPath()) + .append(" ") + .append((FilterUtils.getFilerSymbol(singleSeriesExpression.getFilter()))) + .append(" ") + .append(FilterUtils.getFilterStringValue(singleSeriesExpression.getFilter())); + } + // actual query SQL statement + String realQuerySql; + + realQuerySql = "select * from " + curQueryPath; + if (!(realIotDBCondition.length() == 0)) { + realQuerySql += " where " + realIotDBCondition; + } + realQuerySql += " align by device"; + return queryByConditions( + realQuerySql, database, measurement, tagOrders, fieldOrders, sessionId); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java index 513b06e59f7b..176230f39d91 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java @@ -22,8 +22,9 @@ import java.util.HashMap; import java.util.Map; +import java.util.Set; -public abstract class AbstractInfluxDBMetaManager { +public abstract class AbstractInfluxDBMetaManager implements IInfluxDBMetaManager { protected static final String SELECT_TAG_INFO_SQL = "select database_name,measurement_name,tag_name,tag_order from root.TAG_INFO "; @@ -32,7 +33,8 @@ public abstract class AbstractInfluxDBMetaManager { protected static Map>> database2Measurement2TagOrders = new HashMap<>(); - public static Map getTagOrders(String database, String measurement) { + @Override + public Map getTagOrders(String database, String measurement, long sessionID) { Map tagOrders = new HashMap<>(); Map> measurement2TagOrders = database2Measurement2TagOrders.get(database); @@ -45,8 +47,6 @@ public static Map getTagOrders(String database, String measurem return tagOrders; } - abstract void recover(); - abstract void setStorageGroup(String database, long sessionID); abstract void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID); @@ -69,8 +69,13 @@ public final synchronized Map getTagOrdersWithAutoCreatingSchem return createDatabase(database, sessionID).computeIfAbsent(measurement, m -> new HashMap<>()); } + @Override public final synchronized String generatePath( - String database, String measurement, Map tags, long sessionID) { + String database, + String measurement, + Map tags, + Set fields, + long sessionID) { Map tagKeyToLayerOrders = getTagOrdersWithAutoCreatingSchema(database, measurement, sessionID); // to support rollback if fails to persisting new tag info diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java new file mode 100644 index 000000000000..e3804ca5e76b --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.meta; + +import java.util.Map; +import java.util.Set; + +/** used to manage influxdb metadata */ +public interface IInfluxDBMetaManager { + + void recover(); + + Map getFieldOrders(String database, String measurement, long sessionId); + + String generatePath( + String database, + String measurement, + Map tags, + Set fields, + long sessionID); + + Map getTagOrders(String database, String measurement, long sessionID); +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java index f2e58de977ed..e59d96dfbee3 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java @@ -25,7 +25,10 @@ import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.protocol.influxdb.constant.InfluxConstant; +import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; import org.apache.iotdb.db.qp.Planner; +import org.apache.iotdb.db.qp.physical.PhysicalPlan; import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; import org.apache.iotdb.db.qp.physical.crud.QueryPlan; import org.apache.iotdb.db.qp.physical.sys.SetStorageGroupPlan; @@ -142,6 +145,53 @@ public void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID) } } + @Override + public Map getFieldOrders(String database, String measurement, long sessionID) { + Map fieldOrders = new HashMap<>(); + long queryId = ServiceProvider.SESSION_MANAGER.requestQueryId(true); + try { + String showTimeseriesSql = "show timeseries root." + database + '.' + measurement + ".**"; + PhysicalPlan physicalPlan = + serviceProvider.getPlanner().parseSQLToPhysicalPlan(showTimeseriesSql); + QueryContext queryContext = + serviceProvider.genQueryContext( + queryId, + true, + System.currentTimeMillis(), + showTimeseriesSql, + InfluxConstant.DEFAULT_CONNECTION_TIMEOUT_MS); + QueryDataSet queryDataSet = + serviceProvider.createQueryDataSet( + queryContext, physicalPlan, InfluxConstant.DEFAULT_FETCH_SIZE); + int fieldNums = 0; + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, sessionID); + int tagOrderNums = tagOrders.size(); + while (queryDataSet.hasNext()) { + List fields = queryDataSet.next().getFields(); + String filed = StringUtils.getFieldByPath(fields.get(0).getStringValue()); + if (!fieldOrders.containsKey(filed)) { + // The corresponding order of fields is 1 + tagNum (the first is timestamp, then all tags, + // and finally all fields) + fieldOrders.put(filed, tagOrderNums + fieldNums + 1); + fieldNums++; + } + } + } catch (QueryProcessException + | TException + | StorageEngineException + | SQLException + | IOException + | InterruptedException + | QueryFilterOptimizationException + | MetadataException e) { + throw new InfluxDBException(e.getMessage()); + } finally { + ServiceProvider.SESSION_MANAGER.releaseQueryResourceNoExceptions(queryId); + } + return fieldOrders; + } + private static class InfluxDBMetaManagerHolder { private static final InfluxDBMetaManager INSTANCE = new InfluxDBMetaManager(); diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java new file mode 100644 index 000000000000..abad6b3e0ad0 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.meta; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.schemaregion.SchemaEngineMode; +import org.apache.iotdb.db.service.thrift.impl.ClientRPCServiceImpl; + +public class InfluxDBMetaManagerFactory { + public static IInfluxDBMetaManager getInstance() { + if (IoTDBDescriptor.getInstance() + .getConfig() + .getRpcImplClassName() + .equals(ClientRPCServiceImpl.class.getName())) { + switch (SchemaEngineMode.valueOf( + IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + case Tag: + return TagInfluxDBMetaManager.getInstance(); + default: + return NewInfluxDBMetaManager.getInstance(); + } + } else { + return InfluxDBMetaManager.getInstance(); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java index 5269a2bf443b..10685303c675 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java @@ -19,8 +19,8 @@ package org.apache.iotdb.db.protocol.influxdb.meta; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.db.protocol.influxdb.handler.NewQueryHandler; import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; +import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; import org.apache.iotdb.db.service.thrift.impl.ClientRPCServiceImpl; import org.apache.iotdb.db.service.thrift.impl.NewInfluxDBServiceImpl; import org.apache.iotdb.rpc.IoTDBConnectionException; @@ -57,10 +57,13 @@ public void recover() { try { TSOpenSessionResp tsOpenSessionResp = clientRPCService.openSession( - new TSOpenSessionReq().setUsername("root").setPassword("root")); + new TSOpenSessionReq() + .setUsername("root") + .setPassword("root") + .setZoneId("Asia/Shanghai")); sessionID = tsOpenSessionResp.getSessionId(); TSExecuteStatementResp resp = - NewQueryHandler.executeStatement(SELECT_TAG_INFO_SQL, sessionID); + NewInfluxDBServiceImpl.executeStatement(SELECT_TAG_INFO_SQL, sessionID); IoTDBJDBCDataSet dataSet = QueryResultUtils.creatIoTJDBCDataset(resp); try { Map> measurement2TagOrders; @@ -121,6 +124,29 @@ public void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID) } } + @Override + public Map getFieldOrders(String database, String measurement, long sessionID) { + Map fieldOrders = new HashMap<>(); + String showTimeseriesSql = "show timeseries root." + database + '.' + measurement + ".**"; + TSExecuteStatementResp executeStatementResp = + NewInfluxDBServiceImpl.executeStatement(showTimeseriesSql, sessionID); + List paths = QueryResultUtils.getFullPaths(executeStatementResp); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, sessionID); + int tagOrderNums = tagOrders.size(); + int fieldNums = 0; + for (String path : paths) { + String filed = StringUtils.getFieldByPath(path); + if (!fieldOrders.containsKey(filed)) { + // The corresponding order of fields is 1 + tagNum (the first is timestamp, then all tags, + // and finally all fields) + fieldOrders.put(filed, tagOrderNums + fieldNums + 1); + fieldNums++; + } + } + return fieldOrders; + } + private static class InfluxDBMetaManagerHolder { private static final NewInfluxDBMetaManager INSTANCE = new NewInfluxDBMetaManager(); diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java new file mode 100644 index 000000000000..87fd7da44e7f --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.meta; + +import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; +import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; +import org.apache.iotdb.db.service.thrift.impl.NewInfluxDBServiceImpl; +import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/** use in tag schema region */ +public class TagInfluxDBMetaManager implements IInfluxDBMetaManager { + private static final String STORAGE_GROUP_PATH = "root.influxdbmate"; + + private static final String TAGS_SET = "set.tags"; + + private static final String FIELDS_SET = "set.fields"; + + private TagInfluxDBMetaManager() {} + + public static TagInfluxDBMetaManager getInstance() { + return TagInfluxDBMetaManagerHolder.INSTANCE; + } + + /** use tag schema region to save state information, no need to recover here */ + @Override + public void recover() {} + + /** + * get the fields information of influxdb corresponding database and measurement through tag + * schema region + * + * @param database influxdb database + * @param measurement influxdb measurement + * @param sessionId session id + * @return field information + */ + @Override + public Map getFieldOrders(String database, String measurement, long sessionId) { + return getTimeseriesFieldOrders(database, measurement, FIELDS_SET, sessionId); + } + + /** + * convert the database,measurement,and tags of influxdb to device path of IoTDB,and save the tags + * and fields information of the database and measurement to the tag schema region + * + * @param database influxdb database + * @param measurement influxdb measurement + * @param tags influxdb tags + * @param fields influxdb fields + * @param sessionID session id + * @return device path + */ + @Override + public String generatePath( + String database, + String measurement, + Map tags, + Set fields, + long sessionID) { + createInfluxDBMetaTimeseries(database, measurement, tags, fields, sessionID); + return generateDevicesPath(database, measurement, tags); + } + + private void createInfluxDBMetaTimeseries( + String database, + String measurement, + Map tags, + Set fields, + long sessionID) { + List fieldsList = new ArrayList<>(tags.keySet()); + createInfluxDBMetaTimeseries(database, measurement, TAGS_SET, fieldsList, sessionID); + fieldsList.clear(); + fieldsList.addAll(fields); + createInfluxDBMetaTimeseries(database, measurement, FIELDS_SET, fieldsList, sessionID); + } + + private void createInfluxDBMetaTimeseries( + String database, String measurement, String device, List fields, long sessionID) { + String statement = generateTimeseriesStatement(database, measurement, device, fields); + NewInfluxDBServiceImpl.executeStatement(statement, sessionID); + } + + private String generateTimeseriesStatement( + String database, String measurement, String device, List fields) { + StringBuilder timeseriesStatement = + new StringBuilder( + "create aligned timeseries " + + STORAGE_GROUP_PATH + + ".database." + + database + + ".measurement." + + measurement + + "." + + device + + "("); + for (int i = 0; i < fields.size() - 1; i++) { + String field = fields.get(i); + timeseriesStatement.append(field).append(" BOOLEAN, "); + } + timeseriesStatement.append(fields.get(fields.size() - 1)).append(" BOOLEAN)"); + return timeseriesStatement.toString(); + } + + /** + * get the tags information of influxdb corresponding database and measurement through tag schema + * region + * + * @param database influxdb database + * @param measurement influxdb measurement + * @param sessionID session id + * @return tags information + */ + @Override + public Map getTagOrders(String database, String measurement, long sessionID) { + return getTimeseriesFieldOrders(database, measurement, TAGS_SET, sessionID); + } + + private Map getTimeseriesFieldOrders( + String database, String measurement, String device, long sessionID) { + TSExecuteStatementResp statementResp = + NewInfluxDBServiceImpl.executeStatement( + "show timeseries " + + STORAGE_GROUP_PATH + + ".database." + + database + + ".measurement." + + measurement + + "." + + device, + sessionID); + List timeseriesPaths = QueryResultUtils.getFullPaths(statementResp); + Map fieldOrders = new HashMap<>(); + for (String timeseriesPath : timeseriesPaths) { + String field = StringUtils.getFieldByPath(timeseriesPath); + fieldOrders.put(field, fieldOrders.size()); + } + return fieldOrders; + } + + /** + * convert the database,measurement,and tags of influxdb to device path of IoTDB,ensure that + * influxdb records with the same semantics generate the same device path, so the device path is + * generated in order after sorting the tags + * + * @param database influxdb database + * @param measurement influxdb measurement + * @param tags influxdb tags + * @return device path + */ + private String generateDevicesPath( + String database, String measurement, Map tags) { + TreeMap tagsMap = new TreeMap<>(tags); + tagsMap.put("measurement", measurement); + StringBuilder devicePath = new StringBuilder("root." + database); + for (String tagKey : tagsMap.keySet()) { + devicePath.append(".").append(tagKey).append(".").append(tagsMap.get(tagKey)); + } + return devicePath.toString(); + } + + private static class TagInfluxDBMetaManagerHolder { + private static final TagInfluxDBMetaManager INSTANCE = new TagInfluxDBMetaManager(); + + private TagInfluxDBMetaManagerHolder() {} + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/util/QueryResultUtils.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/util/QueryResultUtils.java index 325971df2598..25199f6429f3 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/util/QueryResultUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/util/QueryResultUtils.java @@ -20,7 +20,7 @@ import org.apache.iotdb.db.protocol.influxdb.constant.InfluxConstant; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; -import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; import org.apache.iotdb.db.query.dataset.AlignByDeviceDataSet; import org.apache.iotdb.rpc.IoTDBJDBCDataSet; import org.apache.iotdb.rpc.StatementExecutionException; @@ -82,7 +82,8 @@ public static QueryResult iotdbResultConvertInfluxResult( QueryResult.Series series = new QueryResult.Series(); series.setName(measurement); // gets the reverse map of the tag - Map tagOrders = InfluxDBMetaManager.getTagOrders(database, measurement); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, -1); Map tagOrderReversed = tagOrders.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); @@ -294,6 +295,12 @@ public static boolean checkQueryResultNull(QueryResult queryResult) { return queryResult.getResults().get(0).getSeries() == null; } + /** + * parse time series paths from query results + * + * @param tsExecuteStatementResp query results + * @return time series paths + */ public static List getFullPaths(TSExecuteStatementResp tsExecuteStatementResp) { List res = new ArrayList<>(); IoTDBJDBCDataSet ioTDBJDBCDataSet = creatIoTJDBCDataset(tsExecuteStatementResp); @@ -309,6 +316,13 @@ public static List getFullPaths(TSExecuteStatementResp tsExecuteStatemen return res; } + /** + * Convert align by device query result of NewIoTDB to the query result of influxdb,used for + * Memory and schema_file schema region + * + * @param tsExecuteStatementResp NewIoTDB execute statement resp to be converted + * @return query results in influxdb format + */ public static QueryResult iotdbResultConvertInfluxResult( TSExecuteStatementResp tsExecuteStatementResp, String database, @@ -321,7 +335,8 @@ public static QueryResult iotdbResultConvertInfluxResult( QueryResult.Series series = new QueryResult.Series(); series.setName(measurement); // gets the reverse map of the tag - Map tagOrders = InfluxDBMetaManager.getTagOrders(database, measurement); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, -1); Map tagOrderReversed = tagOrders.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); @@ -381,6 +396,87 @@ public static QueryResult iotdbResultConvertInfluxResult( return queryResult; } + /** + * Convert align by device query result of NewIoTDB to the query result of influxdb,used for tag + * schema region + * + * @param tsExecuteStatementResp NewIoTDB execute statement resp to be converted + * @return query results in influxdb format + */ + public static QueryResult iotdbResultConvertInfluxResult( + TSExecuteStatementResp tsExecuteStatementResp, + String database, + String measurement, + Map tagOrders, + Map fieldOrders) { + if (tsExecuteStatementResp == null) { + return getNullQueryResult(); + } + // generate series + QueryResult.Series series = new QueryResult.Series(); + series.setName(measurement); + Map tagOrderReversed = + tagOrders.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + int tagSize = tagOrderReversed.size(); + Map fieldOrdersReversed = + fieldOrders.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + ArrayList tagList = new ArrayList<>(); + for (int i = 0; i < tagSize; i++) { + tagList.add(tagOrderReversed.get(i)); + } + + ArrayList fieldList = new ArrayList<>(); + for (int i = 0; i < fieldOrders.size(); i++) { + fieldList.add(fieldOrdersReversed.get(i)); + } + + ArrayList columns = new ArrayList<>(); + columns.add("time"); + columns.addAll(tagList); + columns.addAll(fieldList); + // insert columns into series + series.setColumns(columns); + List> values = new ArrayList<>(); + IoTDBJDBCDataSet ioTDBJDBCDataSet = creatIoTJDBCDataset(tsExecuteStatementResp); + try { + while (ioTDBJDBCDataSet.hasCachedResults()) { + Object[] value = new Object[columns.size()]; + ioTDBJDBCDataSet.constructOneRow(); + value[0] = Long.valueOf(ioTDBJDBCDataSet.getValueByName("Time")); + String deviceName = ioTDBJDBCDataSet.getValueByName("Device"); + String[] deviceNameList = deviceName.split("\\."); + for (int i = 2; i < deviceNameList.length; i += 2) { + if (tagOrders.containsKey(deviceNameList[i])) { + int position = tagOrders.get(deviceNameList[i]) + 1; + value[position] = deviceNameList[i + 1]; + } + } + for (int i = 3; i <= ioTDBJDBCDataSet.columnNameList.size(); i++) { + Object o = ioTDBJDBCDataSet.getObject(ioTDBJDBCDataSet.findColumnNameByIndex(i)); + if (o != null) { + // insert the value of filed into it + int position = fieldOrders.get(ioTDBJDBCDataSet.findColumnNameByIndex(i)) + tagSize + 1; + value[position] = o; + } + } + values.add(Arrays.asList(value)); + } + } catch (Exception e) { + e.printStackTrace(); + } + + series.setValues(values); + + QueryResult queryResult = new QueryResult(); + QueryResult.Result result = new QueryResult.Result(); + result.setSeries(new ArrayList<>(Arrays.asList(series))); + queryResult.setResults(new ArrayList<>(Arrays.asList(result))); + + return queryResult; + } + public static List getInfluxFunctionValues( TSExecuteStatementResp tsExecuteStatementResp) { IoTDBJDBCDataSet ioTDBJDBCDataSet = creatIoTJDBCDataset(tsExecuteStatementResp); diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InfluxDBServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InfluxDBServiceImpl.java index 89e5429dd19a..8209b434876a 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InfluxDBServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InfluxDBServiceImpl.java @@ -151,8 +151,7 @@ public InfluxTSStatus createDatabase(InfluxCreateDatabaseReq req) { public InfluxQueryResultRsp query(InfluxQueryReq req) throws TException { Operator operator = InfluxDBLogicalGenerator.generate(req.command); queryHandler.checkInfluxDBQueryOperator(operator); - return queryHandler.queryInfluxDB( - req.database, (InfluxQueryOperator) operator, req.sessionId, IoTDB.serviceProvider); + return queryHandler.queryInfluxDB(req.database, (InfluxQueryOperator) operator, req.sessionId); } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java index 422bc27fd46a..1c40c94bc7d6 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java @@ -19,17 +19,17 @@ package org.apache.iotdb.db.service.thrift.impl; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.db.protocol.influxdb.constant.InfluxConstant; import org.apache.iotdb.db.protocol.influxdb.dto.IoTDBPoint; import org.apache.iotdb.db.protocol.influxdb.handler.AbstractQueryHandler; -import org.apache.iotdb.db.protocol.influxdb.handler.NewQueryHandler; +import org.apache.iotdb.db.protocol.influxdb.handler.QueryHandlerFactory; import org.apache.iotdb.db.protocol.influxdb.input.InfluxLineParser; -import org.apache.iotdb.db.protocol.influxdb.meta.AbstractInfluxDBMetaManager; -import org.apache.iotdb.db.protocol.influxdb.meta.NewInfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.IInfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; import org.apache.iotdb.db.protocol.influxdb.operator.InfluxQueryOperator; import org.apache.iotdb.db.protocol.influxdb.sql.InfluxDBLogicalGenerator; import org.apache.iotdb.db.protocol.influxdb.util.InfluxReqAndRespUtils; import org.apache.iotdb.db.qp.logical.Operator; -import org.apache.iotdb.db.service.IoTDB; import org.apache.iotdb.db.utils.DataTypeUtils; import org.apache.iotdb.protocol.influxdb.rpc.thrift.InfluxCloseSessionReq; import org.apache.iotdb.protocol.influxdb.rpc.thrift.InfluxCreateDatabaseReq; @@ -42,6 +42,8 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.TSCloseSessionReq; +import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; +import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; import org.apache.iotdb.service.rpc.thrift.TSInsertRecordReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; @@ -57,13 +59,14 @@ public class NewInfluxDBServiceImpl implements IInfluxDBServiceWithHandler { private static final ClientRPCServiceImpl clientRPCService = new ClientRPCServiceImpl(); - private final AbstractInfluxDBMetaManager metaManager; + private final IInfluxDBMetaManager metaManager; private final AbstractQueryHandler queryHandler; public NewInfluxDBServiceImpl() { - metaManager = NewInfluxDBMetaManager.getInstance(); - queryHandler = new NewQueryHandler(); + metaManager = InfluxDBMetaManagerFactory.getInstance(); + metaManager.recover(); + queryHandler = QueryHandlerFactory.getInstance(); } public static ClientRPCServiceImpl getClientRPCService() { @@ -117,8 +120,19 @@ public InfluxTSStatus createDatabase(InfluxCreateDatabaseReq req) { public InfluxQueryResultRsp query(InfluxQueryReq req) throws TException { Operator operator = InfluxDBLogicalGenerator.generate(req.command); queryHandler.checkInfluxDBQueryOperator(operator); - return queryHandler.queryInfluxDB( - req.database, (InfluxQueryOperator) operator, req.sessionId, IoTDB.serviceProvider); + return queryHandler.queryInfluxDB(req.database, (InfluxQueryOperator) operator, req.sessionId); + } + + public static TSExecuteStatementResp executeStatement(String sql, long sessionId) { + TSExecuteStatementReq tsExecuteStatementReq = new TSExecuteStatementReq(); + tsExecuteStatementReq.setStatement(sql); + tsExecuteStatementReq.setSessionId(sessionId); + tsExecuteStatementReq.setStatementId( + NewInfluxDBServiceImpl.getClientRPCService().requestStatementId(sessionId)); + tsExecuteStatementReq.setFetchSize(InfluxConstant.DEFAULT_FETCH_SIZE); + TSExecuteStatementResp executeStatementResp = + NewInfluxDBServiceImpl.getClientRPCService().executeStatement(tsExecuteStatementReq); + return executeStatementResp; } @Override